Cherry-pick GH-836 and GH-725

dev/tasks/java-jars/github.yml

@@ -125,7 +125,7 @@ jobs:
           # Homebrew's python@XXX is updated without "--overwrite", it
           # tries to replace /usr/local/bin/2to3 and so on and causes
           # a conflict error.
-          brew update
+          # brew update
           for python_package in $(brew list | grep python@); do
             brew install --overwrite ${python_package}
           done
@@ -154,8 +154,10 @@ jobs:
           mkdir -p homebrew-custom/Formula
           curl -o homebrew-custom/Formula/cmake.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/f68532bfe5cb87474093df8a839c3818c6aa44dd/Formula/c/cmake.rb
           curl -o homebrew-custom/Formula/boost.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/23f9c56c5075dd56b4471e2c93f89f6400b49ddd/Formula/b/boost.rb
-          brew install -v ./homebrew-custom/Formula/cmake.rb
-          brew install -v ./homebrew-custom/Formula/boost.rb
+          brew tap-new local/homebrew-custom
+          cp ./homebrew-custom/Formula/*.rb "$(brew --repo local/homebrew-custom)/Formula/"
+          brew install -v local/homebrew-custom/cmake
+          brew install -v local/homebrew-custom/boost
           brew pin cmake
           brew pin boost
           #

java/vector/src/main/codegen/includes/vv_imports.ftl

@@ -34,6 +34,7 @@ import org.apache.arrow.vector.complex.*;
 import org.apache.arrow.vector.complex.reader.*;
 import org.apache.arrow.vector.complex.impl.*;
 import org.apache.arrow.vector.complex.writer.*;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ExtensionWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;

java/vector/src/main/codegen/templates/AbstractFieldReader.java

@@ -108,6 +108,27 @@ public void copyAsField(String name, ${name}Writer writer) {
   }
 
   </#list></#list>
+
+  public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory) {
+    fail("CopyAsValue StructWriter");
+  }
+
+  public void read(ExtensionHolder holder) {
+    fail("Extension");
+  }
+
+  public void read(int arrayIndex, ExtensionHolder holder) {
+    fail("RepeatedExtension");
+  }
+
+  public void copyAsValue(AbstractExtensionTypeWriter writer) {
+    fail("CopyAsValueExtension");
+  }
+
+  public void copyAsField(String name, AbstractExtensionTypeWriter writer) {
+    fail("CopyAsFieldExtension");
+  }
+
   public FieldReader reader(String name) {
     fail("reader(String name)");
     return null;

java/vector/src/main/codegen/templates/BaseReader.java

@@ -21,7 +21,7 @@
 
 <#include "/@includes/license.ftl" />
 
-package org.apache.arrow.vector.complex.reader;
+    package org.apache.arrow.vector.complex.reader;
 
 <#include "/@includes/vv_imports.ftl" />
 
@@ -44,21 +44,23 @@ public interface BaseReader extends Positionable{
   public interface StructReader extends BaseReader, Iterable<String>{
     FieldReader reader(String name);
   }
-  
+
   public interface RepeatedStructReader extends StructReader{
     boolean next();
     int size();
     void copyAsValue(StructWriter writer);
+    void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory);
   }
-  
+
   public interface ListReader extends BaseReader{
-    FieldReader reader(); 
+    FieldReader reader();
   }
-  
+
   public interface RepeatedListReader extends ListReader{
     boolean next();
     int size();
     void copyAsValue(ListWriter writer);
+    void copyAsValue(ListWriter writer, ExtensionTypeWriterFactory writerFactory);
   }
 
   public interface MapReader extends BaseReader{
@@ -69,17 +71,17 @@ public interface RepeatedMapReader extends MapReader{
     boolean next();
     int size();
     void copyAsValue(MapWriter writer);
+    void copyAsValue(MapWriter writer, ExtensionTypeWriterFactory writerFactory);
   }
-  
-  public interface ScalarReader extends  
-  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Reader, </#list></#list> 
-  BaseReader {}
-  
+
+  public interface ScalarReader extends
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Reader, </#list></#list>
+  ExtensionReader, BaseReader {}
+
   interface ComplexReader{
     StructReader rootAsStruct();
     ListReader rootAsList();
     boolean rootIsStruct();
     boolean ok();
   }
 }
-

java/vector/src/main/codegen/templates/ComplexCopier.java

@@ -42,10 +42,14 @@ public class ComplexCopier {
    * @param output field to write to
    */
   public static void copy(FieldReader input, FieldWriter output) {
-    writeValue(input, output);
+    writeValue(input, output, null);
   }
 
-  private static void writeValue(FieldReader reader, FieldWriter writer) {
+  public static void copy(FieldReader input, FieldWriter output, ExtensionTypeWriterFactory extensionTypeWriterFactory) {
+    writeValue(input, output, extensionTypeWriterFactory);
+  }
+
+  private static void writeValue(FieldReader reader, FieldWriter writer, ExtensionTypeWriterFactory extensionTypeWriterFactory) {
     final MinorType mt = reader.getMinorType();
 
       switch (mt) {
@@ -61,7 +65,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
             FieldReader childReader = reader.reader();
             FieldWriter childWriter = getListWriterForReader(childReader, writer);
             if (childReader.isSet()) {
-              writeValue(childReader, childWriter);
+              writeValue(childReader, childWriter, extensionTypeWriterFactory);
             } else {
               childWriter.writeNull();
             }
@@ -79,8 +83,8 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
             FieldReader structReader = reader.reader();
             if (structReader.isSet()) {
               writer.startEntry();
-              writeValue(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key()));
-              writeValue(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value()));
+              writeValue(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key()), extensionTypeWriterFactory);
+              writeValue(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value()), extensionTypeWriterFactory);
               writer.endEntry();
             } else {
               writer.writeNull();
@@ -99,7 +103,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
             if (childReader.getMinorType() != Types.MinorType.NULL) {
               FieldWriter childWriter = getStructWriterForReader(childReader, writer, name);
               if (childReader.isSet()) {
-                writeValue(childReader, childWriter);
+                writeValue(childReader, childWriter, extensionTypeWriterFactory);
               } else {
                 childWriter.writeNull();
               }
@@ -110,6 +114,20 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
           writer.writeNull();
         }
         break;
+      case EXTENSIONTYPE:
+        if (extensionTypeWriterFactory == null) {
+          throw new IllegalArgumentException("Must provide ExtensionTypeWriterFactory");
+        }
+        if (reader.isSet()) {
+          Object value = reader.readObject();
+          if (value != null) {
+            writer.addExtensionTypeWriterFactory(extensionTypeWriterFactory);
+            writer.writeExtension(value);
+          }
+        } else {
+          writer.writeNull();
+        }
+        break;
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
   <#assign fields = minor.fields!type.fields />
   <#assign uncappedName = name?uncap_first/>
@@ -162,6 +180,9 @@ private static FieldWriter getStructWriterForReader(FieldReader reader, StructWr
       return (FieldWriter) writer.map(name);
     case LISTVIEW:
       return (FieldWriter) writer.listView(name);
+    case EXTENSIONTYPE:
+      ExtensionWriter extensionWriter = writer.extension(name, reader.getField().getType());
+      return (FieldWriter) extensionWriter;
     default:
       throw new UnsupportedOperationException(reader.getMinorType().toString());
     }
@@ -186,6 +207,9 @@ private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter
       return (FieldWriter) writer.list();
     case LISTVIEW:
       return (FieldWriter) writer.listView();
+    case EXTENSIONTYPE:
+      ExtensionWriter extensionWriter = writer.extension(reader.getField().getType());
+      return (FieldWriter) extensionWriter;
     default:
       throw new UnsupportedOperationException(reader.getMinorType().toString());
     }
@@ -211,6 +235,9 @@ private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter w
         return (FieldWriter) writer.listView();
       case MAP:
         return (FieldWriter) writer.map(false);
+      case EXTENSIONTYPE:
+        ExtensionWriter extensionWriter = writer.extension(reader.getField().getType());
+        return (FieldWriter) extensionWriter;
       default:
         throw new UnsupportedOperationException(reader.getMinorType().toString());
     }

java/vector/src/main/codegen/templates/NullReader.java

@@ -33,12 +33,12 @@
  */
 @SuppressWarnings("unused")
 public class NullReader extends AbstractBaseReader implements FieldReader{
-  
+
   public static final NullReader INSTANCE = new NullReader();
   public static final NullReader EMPTY_LIST_INSTANCE = new NullReader(MinorType.NULL);
   public static final NullReader EMPTY_STRUCT_INSTANCE = new NullReader(MinorType.STRUCT);
   private MinorType type;
-  
+
   private NullReader(){
     super();
     type = MinorType.NULL;
@@ -77,71 +77,73 @@ public void read(Nullable${name}Holder holder){
   public void read(int arrayIndex, ${name}Holder holder){
     throw new ArrayIndexOutOfBoundsException();
   }
-  
+
   public void copyAsValue(${minor.class}Writer writer){}
   public void copyAsField(String name, ${minor.class}Writer writer){}
 
   public void read(int arrayIndex, Nullable${name}Holder holder){
     throw new ArrayIndexOutOfBoundsException();
   }
   </#list></#list>
-
+
+  public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory){}
+  public void read(ExtensionHolder holder) {
+    holder.isSet = 0;
+  }
+
   public int size(){
     return 0;
   }
-  
+
   public boolean isSet(){
     return false;
   }
-  
+
   public boolean next(){
     return false;
   }
-  
+
   public RepeatedStructReader struct(){
     return this;
   }
-  
+
   public RepeatedListReader list(){
     return this;
   }
-  
+
   public StructReader struct(String name){
     return this;
   }
-  
+
   public ListReader list(String name){
     return this;
   }
-  
+
   public FieldReader reader(String name){
     return this;
   }
-  
+
   public FieldReader reader(){
     return this;
   }
-  
+
   private void fail(String name){
     throw new IllegalArgumentException(String.format("You tried to read a %s type when you are using a ValueReader of type %s.", name, this.getClass().getSimpleName()));
   }
-  
+
   <#list ["Object", "BigDecimal", "Short", "Integer", "Long", "Boolean",
           "LocalDateTime", "Duration", "Period", "Double", "Float",
           "Character", "Text", "String", "Byte", "byte[]", "PeriodDuration"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-  
+
   public ${friendlyType} read${safeType}(int arrayIndex){
     return null;
   }
-  
+
   public ${friendlyType} read${safeType}(){
     return null;
   }
   </#list>
-
-}
-
-
 
+}

java/vector/src/main/codegen/templates/PromotableWriter.java

@@ -550,6 +550,10 @@ public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) {
     getWriter(MinorType.EXTENSIONTYPE).addExtensionTypeWriterFactory(factory);
   }
 
+  public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory, ArrowType arrowType) {
+    getWriter(MinorType.EXTENSIONTYPE, arrowType).addExtensionTypeWriterFactory(factory);
+  }
+
   @Override
   public void allocate() {
     getWriter().allocate();

java/vector/src/main/codegen/templates/UnionListWriter.java

@@ -53,6 +53,7 @@ public class Union${listName}Writer extends AbstractFieldWriter {
   private boolean inStruct = false;
   private boolean listStarted = false;
   private String structName;
+  private ArrowType extensionType;
   <#if listName == "LargeList" || listName == "LargeListView">
   private static final long OFFSET_WIDTH = 8;
   <#else>
@@ -203,9 +204,9 @@ public MapWriter map(String name, boolean keysSorted) {
 
   @Override
   public ExtensionWriter extension(ArrowType arrowType) {
+    this.extensionType = arrowType;
     return this;
   }
-
   @Override
   public ExtensionWriter extension(String name, ArrowType arrowType) {
     ExtensionWriter extensionWriter = writer.extension(name, arrowType);
@@ -339,15 +340,14 @@ public void writeExtension(Object value) {
     writer.writeExtension(value);
     writer.setPosition(writer.idx() + 1);
   }
-  
+
   @Override
   public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) {
-    writer.addExtensionTypeWriterFactory(var1);
+    writer.addExtensionTypeWriterFactory(var1, extensionType);
   }
-  
+
   public void write(ExtensionHolder var1) {
     writer.write(var1);
-    writer.setPosition(writer.idx() + 1);
   }
 
   <#list vv.types as type>

java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java

@@ -22,6 +22,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.ReferenceManager;
 import org.apache.arrow.util.Preconditions;
+import org.apache.arrow.vector.complex.impl.ExtensionTypeWriterFactory;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.util.DataSizeRoundingUtil;
 import org.apache.arrow.vector.util.TransferPair;
@@ -248,4 +249,16 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) {
   public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) {
     throw new UnsupportedOperationException();
   }
+
+  @Override
+  public void copyFrom(
+      int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public void copyFromSafe(
+      int fromIndex, int thisIndex, ValueVector from, ExtensionTypeWriterFactory writerFactory) {
+    throw new UnsupportedOperationException();
+  }
 }