Merge branch 'master' of github.com:joyent/java-manta

Author: tjcelaya

.travis.yml

@@ -2,18 +2,36 @@ dist: trusty
 language: java
 # We now cache the Maven dependencies directory for faster builds
 cache:
+  apt: true
   directories:
   - $HOME/.m2
-sudo: false
+  - $HOME/.downloads
+sudo: true
 
 addons:
   apt:
+    sources:
+      - sourceline: 'deb http://repos.azulsystems.com/ubuntu stable main'
+        key_url: 'http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems'
     packages:
-      - oracle-java8-installer
-      - oracle-java8-unlimited-jce-policy
+      - zulu-8
+      - zulu-9
+
+# Oracle JDK 8 and the OpenJDK 8 are both installed by default by TravisCI
+# Additionally, they both have the unlimited JCE policy installed as well.
+# This is why we do not explicitly installed them in our Travis CI
+# configuration.
 
 before_install:
   - echo 'MAVEN_OPTS="-Dorg.slf4j.simpleLogger.defaultLogLevel=warn"' >~/.mavenrc
+  - mkdir -p $HOME/.downloads
+  - wget -N -q -O $HOME/.downloads/ZuluJCEPolicies.zip 'https://cdn.azul.com/zcek/bin/ZuluJCEPolicies.zip'
+  # The SHA256 checksum for JCE for Azul will break if Azul updates their archive.
+  # If so, you will need to update the fingerprint below after verifying the
+  # authenticity of the archive.
+  - echo "8021a28b8cac41b44f1421fd210a0a0822fcaf88d62d2e70a35b2ff628a8675a  $HOME/.downloads/ZuluJCEPolicies.zip" | sha256sum -c
+  - sudo unzip -o -j $HOME/.downloads/ZuluJCEPolicies.zip ZuluJCEPolicies/local_policy.jar ZuluJCEPolicies/US_export_policy.jar -d /usr/lib/jvm/zulu-8-amd64/jre/lib/security
+  - sudo unzip -o -j $HOME/.downloads/ZuluJCEPolicies.zip ZuluJCEPolicies/local_policy.jar ZuluJCEPolicies/US_export_policy.jar -d /usr/lib/jvm/zulu-9-amd64/lib/security
 
 matrix:
   fast_finish: true
@@ -36,5 +54,22 @@ matrix:
         - DESC="openjdk8 unit tests"
         - CMD="mvn clean test -Dcheckstyle.skip=true"
         - LANG=en_US.utf8
+    # unit tests (zulu-8)
+    - env:
+        - JAVA_HOME=/usr/lib/jvm/zulu-8-amd64
+        - DESC="zulu-8 unit tests"
+        - CMD="mvn clean test -Dcheckstyle.skip=true"
+        - LANG=en_US.utf8
+    - jdk: oraclejdk9
+      env:
+        - DESC="oraclejdk9 unit tests"
+        - CMD="mvn clean test -Dcheckstyle.skip=true"
+        - LANG=en_US.utf8
+    # unit tests (zulu-9)
+    - env:
+        - JAVA_HOME=/usr/lib/jvm/zulu-9-amd64
+        - DESC="zulu-9 unit tests"
+        - CMD="mvn clean test -Dcheckstyle.skip=true"
+        - LANG=en_US.utf8
 
 script: echo ${CMD}; ${CMD}

USAGE.md

@@ -93,6 +93,7 @@ Below is a table of available configuration parameters followed by detailed desc
 | manta.tcp_socket_timeout           | MANTA_TCP_SOCKET_TIMEOUT       | 10000                                |                          |
 | manta.connection_request_timeout   | MANTA_CONNECTION_REQUEST_TIMEOUT | 1000                               |                          |
 | manta.upload_buffer_size           | MANTA_UPLOAD_BUFFER_SIZE       | 16384                                |                          |
+| manta.skip_directory_depth         | MANTA_SKIP_DIRECTORY_DEPTH     |                                      |                          |
 | manta.client_encryption            | MANTA_CLIENT_ENCRYPTION        | false                                |                          |
 | manta.encryption_key_id            | MANTA_CLIENT_ENCRYPTION_KEY_ID |                                      |                          |
 | manta.encryption_algorithm         | MANTA_ENCRYPTION_ALGORITHM     | AES128/CTR/NoPadding                 |                          |
@@ -148,6 +149,12 @@ Note: Dynamic Updates marked with an asterisk (*) are enabled by the `AuthAwareC
     The initial amount of bytes to attempt to load into memory when uploading a stream. If the
     entirety of the stream fits within the number of bytes of this value, then the
     contents of the buffer are directly uploaded to Manta in a retryable form.
+* `manta.skip_directory_depth` (**MANTA_SKIP_DIRECTORY_DEPTH**)
+    Integer indicating the number of directory levels to attempt to skip when performing a recursive `putDirectory`
+    operation. Set to 0 to disable the optimization entirely. Irrelevant when the depth of the recursive `putDirectory`
+    call is less than the setting. When creating a directory with more levels than the setting, the client will attempt
+    to skip this many non-system directories from the root. Will return to normal directory creation procedure if
+    the skipped `PUT` fails or proceed creating all directories between the skip depth and the child on success.
 * `manta.client_encryption` (**MANTA_CLIENT_ENCRYPTION**)
     Boolean indicating if client-side encryption is enabled.
 * `manta.encryption_key_id` (**MANTA_CLIENT_ENCRYPTION_KEY_ID**)

checkstyle.xml

@@ -82,7 +82,9 @@
             <property name="excludeScope" value="package"/>
         </module>
         <module name="JavadocType" />
-        <module name="JavadocVariable"/>
+        <module name="JavadocVariable">
+            <property name="ignoreNamePattern" value="LOG|LOGGER"/>
+        </module>
         <module name="JavadocStyle" />
 
         <!-- Checks for Naming Conventions.                  -->

java-manta-client-kryo-serialization/src/main/java/com/joyent/manta/serialization/BaseBlockCipherSerializer.java

@@ -25,13 +25,14 @@
 import org.bouncycastle.jcajce.provider.symmetric.util.BlockCipherProvider;
 import org.bouncycastle.jcajce.spec.GOST28147ParameterSpec;
 import org.bouncycastle.jcajce.util.BCJcaJceHelper;
-import org.objenesis.instantiator.sun.MagicInstantiator;
+import org.objenesis.instantiator.sun.UnsafeFactoryInstantiator;
 
 import javax.crypto.spec.GCMParameterSpec;
 import javax.crypto.spec.IvParameterSpec;
 import javax.crypto.spec.PBEParameterSpec;
 import javax.crypto.spec.RC2ParameterSpec;
 import javax.crypto.spec.RC5ParameterSpec;
+
 import java.lang.reflect.Field;
 import java.security.AlgorithmParameters;
 
@@ -141,25 +142,25 @@ private void registerClasses(final Kryo kryo) {
         kryo.register(BufferedBlockCipher.class);
         kryo.register(bufferedGenericBlockCipherClass,
                 new CompatibleFieldSerializer<>(kryo, bufferedGenericBlockCipherClass))
-                .setInstantiator(new MagicInstantiator<>(bufferedGenericBlockCipherClass));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(bufferedGenericBlockCipherClass));
         kryo.register(SICBlockCipher.class)
-                .setInstantiator(new MagicInstantiator<>(SICBlockCipher.class));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(SICBlockCipher.class));
         kryo.register(AlgorithmParameters.class);
         kryo.register(BCJcaJceHelper.class);
         kryo.register(ParametersWithIV.class)
-                .setInstantiator(new MagicInstantiator<>(ParametersWithIV.class));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(ParametersWithIV.class));
         kryo.register(KeyParameter.class)
-                .setInstantiator(new MagicInstantiator<>(KeyParameter.class));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(KeyParameter.class));
         kryo.register(paddedBufferedBlockCipherClass)
-                .setInstantiator(new MagicInstantiator<>(paddedBufferedBlockCipherClass));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(paddedBufferedBlockCipherClass));
         kryo.register(CBCBlockCipher.class)
-                .setInstantiator(new MagicInstantiator<>(CBCBlockCipher.class));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(CBCBlockCipher.class));
         kryo.register(AEADParameters.class, new CompatibleFieldSerializer<>(kryo, AEADParameters.class))
-                .setInstantiator(new MagicInstantiator<>(AEADParameters.class));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(AEADParameters.class));
         kryo.register(aeadGenericBlockCipherClass, new CompatibleFieldSerializer<>(kryo, aeadGenericBlockCipherClass))
-                .setInstantiator(new MagicInstantiator<>(aeadGenericBlockCipherClass));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(aeadGenericBlockCipherClass));
         kryo.register(GCMBlockCipher.class, new CompatibleFieldSerializer<>(kryo, GCMBlockCipher.class))
-                .setInstantiator(new MagicInstantiator<>(GCMBlockCipher.class));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(GCMBlockCipher.class));
         kryo.register(RC2ParameterSpec.class);
         kryo.register(RC5ParameterSpec.class);
         kryo.register(GCMParameterSpec.class);

java-manta-client-kryo-serialization/src/main/java/com/joyent/manta/serialization/CipherSerializer.java

@@ -14,6 +14,7 @@
 import com.esotericsoftware.kryo.serializers.JavaSerializer;
 import org.bouncycastle.jcajce.provider.symmetric.AES;
 import org.objenesis.instantiator.sun.MagicInstantiator;
+import org.objenesis.instantiator.sun.UnsafeFactoryInstantiator;
 
 import javax.crypto.Cipher;
 import javax.crypto.spec.SecretKeySpec;
@@ -94,10 +95,10 @@ private void registerClasses(final Kryo kryo) {
         kryo.register(IdentityHashMap.class, new JavaSerializer());
 
         kryo.register(java.lang.ref.WeakReference.class)
-                .setInstantiator(new MagicInstantiator<>(java.lang.ref.WeakReference.class));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(java.lang.ref.WeakReference.class));
 
         kryo.register(java.security.AlgorithmParameters.class)
-                .setInstantiator(new MagicInstantiator<>(java.security.AlgorithmParameters.class));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(java.security.AlgorithmParameters.class));
 
         kryo.register(SecretKeySpec.class, new JavaSerializer());
 
@@ -142,7 +143,7 @@ private void registerWithCompatSerializerMagicInstantiator(final Kryo kryo, fina
 
         if (clazz != null) {
             kryo.register(clazz, new CompatibleFieldSerializer<>(kryo, clazz))
-                    .setInstantiator(new MagicInstantiator<>(clazz));
+                    .setInstantiator(new UnsafeFactoryInstantiator<>(clazz));
         }
     }
 
@@ -170,7 +171,7 @@ private void registerWithMagicInstantiator(final Kryo kryo, final String classNa
         Class<?> clazz = findClass(className);
 
         if (clazz != null) {
-            kryo.register(clazz).setInstantiator(new MagicInstantiator<>(clazz));
+            kryo.register(clazz).setInstantiator(new UnsafeFactoryInstantiator<>(clazz));
         }
     }
 

java-manta-client-kryo-serialization/src/main/java/com/joyent/manta/serialization/EncryptionStateSerializer.java

@@ -21,7 +21,7 @@
 import com.joyent.manta.util.HmacOutputStream;
 import org.bouncycastle.crypto.macs.HMac;
 import org.bouncycastle.jcajce.io.CipherOutputStream;
-import org.objenesis.instantiator.sun.MagicInstantiator;
+import org.objenesis.instantiator.sun.UnsafeFactoryInstantiator;
 
 import javax.crypto.Cipher;
 import javax.crypto.SecretKey;
@@ -72,7 +72,7 @@ public EncryptionStateSerializer(final Kryo kryo,
      */
     private void registerClasses(final Kryo kryo) {
         kryo.register(EncryptionContext.class)
-                .setInstantiator(new MagicInstantiator<>(EncryptionContext.class));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(EncryptionContext.class));
         kryo.register(SupportedCipherDetails.class, new SupportedCipherDetailsSerializer());
         kryo.register(AesCtrCipherDetails.class, new SupportedCipherDetailsSerializer());
         kryo.register(AesCbcCipherDetails.class, new SupportedCipherDetailsSerializer());
@@ -85,9 +85,9 @@ private void registerClasses(final Kryo kryo) {
                 "CloseShieldOutputStream reference must not be null");
 
         kryo.register(closeShieldStreamClass)
-                .setInstantiator(new MagicInstantiator<>(closeShieldStreamClass));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(closeShieldStreamClass));
         kryo.register(CipherOutputStream.class)
-                .setInstantiator(new MagicInstantiator<>(CipherOutputStream.class));
+                .setInstantiator(new UnsafeFactoryInstantiator<>(CipherOutputStream.class));
     }
 
     /**

java-manta-client-kryo-serialization/src/main/java/com/joyent/manta/serialization/ReflectionUtils.java

@@ -115,6 +115,7 @@ static Field getField(final Class<?> clazz, final String name) {
      * @param object object to read from
      * @return field's value
      */
+    @SuppressWarnings("deprecation")
     static Object readField(final Field field, final Object object) {
         Objects.requireNonNull(field, "Field must not be null");
         Objects.requireNonNull(object, "Object must not be null");
@@ -142,6 +143,7 @@ static Object readField(final Field field, final Object object) {
      * @param target target object
      * @param value object to write
      */
+    @SuppressWarnings("deprecation")
     static void writeField(final Field field, final Object target, final Object value) {
         Objects.requireNonNull(field, "Field must not be null");
         Objects.requireNonNull(target, "Target must not be null");

java-manta-client-unshaded/src/main/java/com/joyent/manta/client/MantaClient.java

@@ -79,7 +79,6 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.time.Instant;
 import java.time.temporal.TemporalAmount;
@@ -1576,13 +1575,14 @@ public void putDirectory(final String path, final boolean recursive)
     /**
      * Creates a directory in Manta.
      *
-     * @param rawPath The fully qualified path of the Manta directory.
+     * @param rawPath   The fully qualified path of the Manta directory.
      * @param recursive recursive create all of the directories specified in the path
-     * @param headers Optional {@link MantaHttpHeaders}. Consult the Manta api for more header information.
-     * @throws IOException If an IO exception has occurred.
+     * @param headers   Optional {@link MantaHttpHeaders}. Consult the Manta api for more header information.
+     * @throws IOException                      If an IO exception has occurred.
      * @throws MantaClientHttpResponseException If a http status code {@literal > 300} is returned.
      */
-    public void putDirectory(final String rawPath, final boolean recursive,
+    public void putDirectory(final String rawPath,
+                             final boolean recursive,
                              final MantaHttpHeaders headers)
             throws IOException {
         Validate.notBlank(rawPath, "rawPath must not be blank");
@@ -1592,23 +1592,12 @@ public void putDirectory(final String rawPath, final boolean recursive,
             return;
         }
 
-        final String[] parts = rawPath.split(SEPARATOR);
-        final Iterator<Path> itr = Paths.get("", parts).iterator();
-        final StringBuilder sb = new StringBuilder(SEPARATOR);
-
-        for (int i = 0; itr.hasNext(); i++) {
-            final String part = itr.next().toString();
-            sb.append(part);
-
-            // This means we aren't in the home nor in the reserved
-            // directory path (stor, public, jobs, etc)
-            if (i > 1) {
-                putDirectory(sb.toString(), headers);
-            }
-
-            if (itr.hasNext()) {
-                sb.append(SEPARATOR);
-            }
+        final Integer skipDepth = config.getSkipDirectoryDepth();
+        final RecursiveDirectoryCreationStrategy directoryCreationStrategy;
+        if (skipDepth != null && 0 < skipDepth) {
+            RecursiveDirectoryCreationStrategy.createWithSkipDepth(this, rawPath, headers, skipDepth);
+        } else {
+            RecursiveDirectoryCreationStrategy.createCompletely(this, rawPath, headers);
         }
     }
 

java-manta-client-unshaded/src/main/java/com/joyent/manta/client/RecursiveDirectoryCreationStrategy.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2017, Joyent, Inc. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+package com.joyent.manta.client;
+
+import com.joyent.manta.exception.MantaClientHttpResponseException;
+import com.joyent.manta.exception.MantaErrorCode;
+import com.joyent.manta.http.MantaHttpHeaders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static com.joyent.manta.util.MantaUtils.writeablePrefixPaths;
+
+/**
+ * Utility class for recursive directory creation strategies.
+ *
+ * @author <a href="https://github.com/tjcelaya">Tomas Celayac</a>
+ * @since 3.1.7
+ */
+final class RecursiveDirectoryCreationStrategy {
+
+    private static final Logger LOG = LoggerFactory.getLogger(RecursiveDirectoryCreationStrategy.class);
+
+    private RecursiveDirectoryCreationStrategy() {
+    }
+
+    static long createWithSkipDepth(final MantaClient client,
+                                    final String rawPath,
+                                    final MantaHttpHeaders headers,
+                                    final int skipDepth) throws IOException {
+        final String[] paths = writeablePrefixPaths(rawPath);
+
+        if (paths.length <= skipDepth) {
+            return createCompletely(client, rawPath, headers);
+        }
+
+        final String assumedExistingDirectory = paths[skipDepth - 1];
+        final String maybeNewDirectory = paths[skipDepth];
+
+        LOG.debug("ASSUME {}", assumedExistingDirectory);
+
+        final Boolean redundantPut = createNewDirectory(client, maybeNewDirectory, headers, rawPath);
+        long ops = 1;
+
+        if (redundantPut == null) {
+            LOG.debug("FAILED {}", maybeNewDirectory);
+
+            // failed to create directory at the skip depth, proceed normally
+            return ops + createCompletely(client, rawPath, headers);
+        }
+
+        for (int idx = skipDepth + 1; idx < paths.length; idx++) {
+            client.putDirectory(paths[idx], headers);
+            ops++;
+        }
+
+        return ops;
+    }
+
+    static long createCompletely(final MantaClient client,
+                                 final String rawPath,
+                                 final MantaHttpHeaders headers) throws IOException {
+        long ops = 0;
+        for (final String path : writeablePrefixPaths(rawPath)) {
+            client.putDirectory(path, headers);
+            ops++;
+        }
+
+        return ops;
+    }
+
+    /**
+     * Try to create a directory and unpack the error. The Boolean is intentional and acts as a tri-state variable.
+     *
+     * NULL  = creation failed.
+     * TRUE  = a new directory was actually created
+     * FALSE = the directory already existed
+     *
+     * @return whether or not the directory was actually new, or null if it failed to be created
+     */
+    private static Boolean createNewDirectory(final MantaClient client,
+                                              final String path,
+                                              final MantaHttpHeaders headers,
+                                              final String targetPath) throws IOException {
+        try {
+            return client.putDirectory(path, headers);
+        } catch (final MantaClientHttpResponseException mchre) {
+            if (mchre.getServerCode().equals(MantaErrorCode.DIRECTORY_DOES_NOT_EXIST_ERROR)) {
+                return null;
+            } else {
+                mchre.setContextValue("recursiveDirectoryCreationTarget", targetPath);
+                throw mchre;
+            }
+        }
+    }
+}