[PECOBLR-1121] Arrow patch to circumvent Arrow issues with JDk 16+

.github/workflows/bugCatcher.yml

@@ -30,7 +30,7 @@ jobs:
           restore-keys: ${{ runner.os }}-m2
 
       - name: Run Integration Tests
-        run: mvn -B test -Dtest=*e2e/OAuthTests*
+        run: mvn -pl jdbc-core -B test -Dtest='*e2e/OAuthTests*'
         env:
           DATABRICKS_HOST: ${{ secrets.JDBC_PAT_TEST_HOST_NAME }}
           DATABRICKS_HTTP_PATH: ${{ secrets.JDBC_PAT_TEST_HTTP_PATH }}

.github/workflows/concurrencyExecutionTests.yml

@@ -48,7 +48,7 @@ jobs:
           restore-keys: ${{ runner.os }}-m2
 
       - name: Run Concurrency Execution Tests
-        run: mvn -B test -Dtest=com.databricks.jdbc.integration.e2e.ConcurrentExecutionTests -DargLine="-ea"
+        run: mvn -pl jdbc-core -B test -Dtest=com.databricks.jdbc.integration.e2e.ConcurrentExecutionTests -DargLine="-ea"
         env:
           DATABRICKS_TOKEN: ${{ secrets.JDBC_PAT_TEST_TOKEN }}
           DATABRICKS_USER: ${{ secrets.DATABRICKS_USER }}

.github/workflows/coverageReport.yml

@@ -34,7 +34,7 @@ jobs:
           ${{ runner.os }}-m2-
 
     - name: Run tests with coverage
-      run: mvn clean test jacoco:report
+      run: mvn -pl jdbc-core clean test -Dgroups='!Jvm17PlusAndArrowToNioReflectionDisabled' jacoco:report
 
     - name: Check for coverage override
       id: override
@@ -53,7 +53,7 @@ jobs:
     - name: Check coverage percentage
       if: steps.override.outputs.override == 'false'
       run: |
-        COVERAGE_FILE="target/site/jacoco/jacoco.xml"
+        COVERAGE_FILE="jdbc-core/target/site/jacoco/jacoco.xml"
         if [ ! -f "$COVERAGE_FILE" ]; then
           echo "ERROR: Coverage file not found at $COVERAGE_FILE"
           exit 1

.github/workflows/loggingTesting.yml

@@ -45,13 +45,13 @@ jobs:
       - name: Find JAR file
         shell: bash
         run: |
-          # Find the main JAR file dynamically (fat JAR, not thin, not tests)
-          MAIN_JAR=$(find target -maxdepth 1 -name "databricks-jdbc-*.jar" \
+          # Find the main JAR file dynamically (uber JAR from assembly-uber module)
+          MAIN_JAR=$(find assembly-uber/target -maxdepth 1 -name "databricks-jdbc-*.jar" \
             -not -name "*-thin.jar" \
             -not -name "*-tests.jar" | head -1)
           if [ -z "$MAIN_JAR" ]; then
-            echo "ERROR: Could not find main JAR file in target directory"
-            ls -la target/
+            echo "ERROR: Could not find main JAR file in assembly-uber/target directory"
+            ls -la assembly-uber/target/
             exit 1
           fi
           echo "Using JAR file: $MAIN_JAR"
@@ -88,18 +88,18 @@ jobs:
       - name: Clean & Compile LoggingTest
         shell: bash
         run: |
-          rm -rf target/test-classes
-          mkdir -p target/test-classes
+          rm -rf jdbc-core/target/test-classes
+          mkdir -p jdbc-core/target/test-classes
 
           echo "Using JAR file: $MAIN_JAR"
 
           javac \
             -cp "$MAIN_JAR" \
-            -d target/test-classes \
-            src/test/java/com/databricks/client/jdbc/LoggingTest.java
+            -d jdbc-core/target/test-classes \
+            jdbc-core/src/test/java/com/databricks/client/jdbc/LoggingTest.java
 
           echo "==== Checking compiled classes ===="
-          find target/test-classes -type f
+          find jdbc-core/target/test-classes -type f
 
       - name: Run LoggingTest
         shell: bash
@@ -110,7 +110,7 @@ jobs:
           echo "Using classpath separator: '$SEP'"
           echo "Using JAR file: $MAIN_JAR"
 
-          CP="target/test-classes${SEP}$MAIN_JAR"
+          CP="jdbc-core/target/test-classes${SEP}$MAIN_JAR"
 
           java \
             --add-opens=java.base/java.nio=ALL-UNNAMED \

.github/workflows/prCheck.yml

@@ -76,21 +76,51 @@ jobs:
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
           restore-keys: ${{ runner.os }}-m2
 
+      - name: Set up Maven Toolchains
+        shell: bash
+        run: |
+          mkdir -p ~/.m2
+          cat > ~/.m2/toolchains.xml <<EOF
+          <?xml version="1.0" encoding="UTF-8"?>
+          <toolchains>
+            <toolchain>
+              <type>jdk</type>
+              <provides>
+                <version>${{ matrix.java-version }}</version>
+              </provides>
+              <configuration>
+                <jdkHome>$JAVA_HOME</jdkHome>
+              </configuration>
+            </toolchain>
+          </toolchains>
+          EOF
+
+      - name: Check Arrow Patch Tests
+        shell: bash
+        if: matrix.java-version >= 17
+        run: mvn -Pjdk${{ matrix.java-version }}-NioNotOpen -pl jdbc-core test -Dgroups='Jvm17PlusAndArrowToNioReflectionDisabled'
+
+      - name: Check Arrow Allocator Manager Tests
+        shell: bash
+        if: matrix.java-version >= 17
+        run: mvn -Pjdk${{ matrix.java-version }}-NioNotOpen -pl jdbc-core test -Dgroups='Jvm17PlusAndArrowToNioReflectionDisabled' -Dtest="ArrowBufferAllocatorNettyManagerTest,ArrowBufferAllocatorUnsafeManagerTest,ArrowBufferAllocatorUnknownManagerTest" -DforkCount=1 -DreuseForks=false
+
+      - name: Check Arrow Memory Tests
+        shell: bash
+        run: mvn -Plow-memory -pl jdbc-core test -Dtest='DatabricksArrowPatchMemoryUsageTest'
+
       - name: Check Unit Tests
         shell: bash
-        run: mvn test -Dtest='!**/integration/**,!**/DatabricksDriverExamples.java,!**/ProxyTest.java,!**/LoggingTest.java,!**/SSLTest.java'
+        run: mvn -pl jdbc-core clean test -Dgroups='!Jvm17PlusAndArrowToNioReflectionDisabled' jacoco:report
 
       - name: Install xmllint
         if: runner.os == 'Linux'
         run: sudo apt-get update && sudo apt-get install -y libxml2-utils
 
-      - name: JaCoCo report
-        run: mvn --batch-mode --errors jacoco:report --file pom.xml
-
       - name: Extract codeCov percentage
         shell: bash
         run: |
-          COVERAGE_FILE="target/site/jacoco/jacoco.xml"
+          COVERAGE_FILE="jdbc-core/target/site/jacoco/jacoco.xml"
           COVERED=$(xmllint --xpath "string(//report/counter[@type='INSTRUCTION']/@covered)" "$COVERAGE_FILE")
           MISSED=$(xmllint --xpath "string(//report/counter[@type='INSTRUCTION']/@missed)" "$COVERAGE_FILE")
           TOTAL=$((COVERED + MISSED))
@@ -114,4 +144,80 @@ jobs:
             exit 1
           else
             echo "Coverage is equal to or greater than 85%"
-          fi
+          fi
+
+  packaging-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        java-version: [ 17 ]
+        github-runner: [ linux-ubuntu-latest, windows-server-latest ]
+
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: ${{ matrix.github-runner }}
+
+    steps:
+      - name: Set up JDK ${{ matrix.java-version }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'adopt'
+
+      - name: Enable long paths
+        if: runner.os == 'Windows'
+        run: git config --system core.longpaths true
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
+          repository: ${{ github.event.pull_request.head.repo.full_name ||  github.repository }}
+
+      - name: Cache Maven packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+
+      - name: Set up Maven Toolchains
+        shell: bash
+        run: |
+          mkdir -p ~/.m2
+          cat > ~/.m2/toolchains.xml <<EOF
+          <?xml version="1.0" encoding="UTF-8"?>
+          <toolchains>
+            <toolchain>
+              <type>jdk</type>
+              <provides>
+                <version>${{ matrix.java-version }}</version>
+              </provides>
+              <configuration>
+                <jdkHome>$JAVA_HOME</jdkHome>
+              </configuration>
+            </toolchain>
+          </toolchains>
+          EOF
+
+      - name: Install JDBC artifacts into maven local
+        shell: bash
+        run: mvn -B -pl jdbc-core,assembly-uber,assembly-thin install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -Ddependency-check.skip=true
+
+      - name: Check Uber Jar Packaging
+        shell: bash
+        run: mvn -pl test-assembly-uber test
+        env:
+          DATABRICKS_HOST: ${{ secrets.JDBC_PAT_TEST_HOST_NAME }}
+          DATABRICKS_HTTP_PATH: ${{ secrets.JDBC_PAT_TEST_HTTP_PATH }}
+          DATABRICKS_USER: ${{ secrets.DATABRICKS_USER }}
+          DATABRICKS_TOKEN: ${{ secrets.JDBC_PAT_TEST_TOKEN }}
+
+      - name: Check Thin Jar Packaging
+        shell: bash
+        run:  mvn -pl test-assembly-thin test
+        env:
+          DATABRICKS_HOST: ${{ secrets.JDBC_PAT_TEST_HOST_NAME }}
+          DATABRICKS_HTTP_PATH: ${{ secrets.JDBC_PAT_TEST_HTTP_PATH }}
+          DATABRICKS_USER: ${{ secrets.DATABRICKS_USER }}
+          DATABRICKS_TOKEN: ${{ secrets.JDBC_PAT_TEST_TOKEN }}

.github/workflows/prCheckJDK8.yml

@@ -48,4 +48,8 @@ jobs:
           restore-keys: ${{ runner.os }}-jdk8-m2
 
       - name: Run Unit Tests
-        run: mvn clean test
+        run: mvn -pl jdbc-core clean test -Dgroups='!Jvm17PlusAndArrowToNioReflectionDisabled'
+
+      - name: Check Arrow Memory Tests
+        shell: bash
+        run: mvn -Plow-memory -pl jdbc-core test -Dtest='DatabricksArrowPatchMemoryUsageTest'

.github/workflows/prIntegrationTests.yml

@@ -16,10 +16,10 @@ jobs:
         include:
           # SQL_EXEC mode: Tests SEA client behavior
           # Note: CircuitBreakerIntegrationTests requires THRIFT_SERVER mode (tested in second matrix entry)
-          - test-command: mvn -B compile test -Dtest=*IntegrationTests,!M2MPrivateKeyCredentialsIntegrationTests,!M2MAuthIntegrationTests,!CircuitBreakerIntegrationTests,!ThriftCloudFetchFakeIntegrationTests
+          - test-command: mvn -pl jdbc-core -B compile test -Dtest=*IntegrationTests,!M2MPrivateKeyCredentialsIntegrationTests,!M2MAuthIntegrationTests,!CircuitBreakerIntegrationTests,!ThriftCloudFetchFakeIntegrationTests
             fake-service-type: 'SQL_EXEC'
           # THRIFT_SERVER mode: Tests Thrift client behavior and circuit breaker fallback
-          - test-command: mvn -B compile test -Dtest=*IntegrationTests,!M2MPrivateKeyCredentialsIntegrationTests,!SqlExecApiHybridResultsIntegrationTests,!DBFSVolumeIntegrationTests,!M2MAuthIntegrationTests,!UCVolumeIntegrationTests,!SqlExecApiIntegrationTests
+          - test-command: mvn -pl jdbc-core -B compile test -Dtest=*IntegrationTests,!M2MPrivateKeyCredentialsIntegrationTests,!SqlExecApiHybridResultsIntegrationTests,!DBFSVolumeIntegrationTests,!M2MAuthIntegrationTests,!UCVolumeIntegrationTests,!SqlExecApiIntegrationTests
             fake-service-type: 'THRIFT_SERVER'
     steps:
       - name: Checkout PR

.github/workflows/proxyTesting.yml

@@ -157,7 +157,7 @@ jobs:
       ################################################################
       - name: Run ProxyTest
         run: |
-          mvn test -Dtest=**/ProxyTest.java
+          mvn -pl jdbc-core test -Dtest=**/ProxyTest.java
 
       ################################################################
       # 14) Cleanup

.github/workflows/release-thin.yml

@@ -31,111 +31,37 @@ jobs:
 
       - name: Set up Java for publishing to Maven Central Repository
         uses: actions/setup-java@v4
+        env:
+          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
         with:
           java-version: 11
-          distribution: "adopt"
           server-id: central
+          distribution: "adopt"
           server-username: MAVEN_CENTRAL_USERNAME
           server-password: MAVEN_CENTRAL_PASSWORD
           gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
           gpg-passphrase: GPG_PASSPHRASE
 
-      - name: Configure GPG
+      # Step 1: Build and install dependencies to local Maven repository
+      # This builds jdbc-core (and parent) without publishing them.
+      # The -am flag builds all dependencies needed by assembly-thin.
+      # We use -Prelease here to generate sources/javadoc JARs for jdbc-core,
+      # which assembly-thin needs for its own sources/javadoc artifacts.
+      # GPG signing is skipped since we're only installing locally, not publishing.
+      - name: Build dependencies
         run: |
-          echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf
-          echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
-          gpg-connect-agent reloadagent /bye
+          mvn -Prelease clean install --batch-mode -pl jdbc-core -am -Dgpg.skip=true
 
-      - name: Build thin JAR with sources and javadocs
+      # Step 2: Deploy only the thin JAR module to Maven Central
+      # We don't use -am here to avoid the central-publishing-maven-plugin
+      # from collecting parent/jdbc-core artifacts into the deployment bundle.
+      # The jdbc-core dependency is already available from Step 1.
+      - name: Publish thin JAR to Maven Central
         run: |
-          # Build main artifacts including sources and javadocs
-          mvn -B -DskipTests package source:jar javadoc:jar
-
-      - name: Sign all thin JAR artifacts
-        run: |
-          VERSION=$(grep -m1 '<version>' pom.xml | sed 's/.*<version>\(.*\)<\/version>.*/\1/')
-
-          # Sign thin JAR
-          echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --pinentry-mode loopback \
-            --armor --detach-sign "target/databricks-jdbc-${VERSION}-thin.jar"
-
-          # Sign sources JAR
-          echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --pinentry-mode loopback \
-            --armor --detach-sign "target/databricks-jdbc-${VERSION}-sources.jar"
-
-          # Sign javadoc JAR
-          echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --pinentry-mode loopback \
-            --armor --detach-sign "target/databricks-jdbc-${VERSION}-javadoc.jar"
-        env:
-          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
-
-      - name: Verify all required artifacts exist
-        run: |
-          VERSION=$(grep -m1 '<version>' pom.xml | sed 's/.*<version>\(.*\)<\/version>.*/\1/')
-          test -f "target/databricks-jdbc-${VERSION}-thin.jar"
-          test -f "target/databricks-jdbc-${VERSION}-thin.jar.asc"
-          test -f "target/databricks-jdbc-${VERSION}-sources.jar"
-          test -f "target/databricks-jdbc-${VERSION}-sources.jar.asc"
-          test -f "target/databricks-jdbc-${VERSION}-javadoc.jar"
-          test -f "target/databricks-jdbc-${VERSION}-javadoc.jar.asc"
-
-      - name: Publish Thin JAR as Separate Artifact to Maven Central
-        run: |
-          VERSION=$(grep -m1 '<version>' pom.xml | sed 's/.*<version>\(.*\)<\/version>.*/\1/')
-
-          echo "Creating deployment bundle for thin JAR..."
-
-          # Create staging directory
-          mkdir -p target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}
-
-          # Copy thin JAR and its signature
-          cp "target/databricks-jdbc-${VERSION}-thin.jar" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}.jar
-          cp "target/databricks-jdbc-${VERSION}-thin.jar.asc" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}.jar.asc
-
-          # Copy sources JAR and its signature
-          cp "target/databricks-jdbc-${VERSION}-sources.jar" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}-sources.jar
-          cp "target/databricks-jdbc-${VERSION}-sources.jar.asc" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}-sources.jar.asc
-
-          # Copy javadoc JAR and its signature
-          cp "target/databricks-jdbc-${VERSION}-javadoc.jar" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}-javadoc.jar
-          cp "target/databricks-jdbc-${VERSION}-javadoc.jar.asc" \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}-javadoc.jar.asc
-
-          # Copy POM and sign it
-          cp thin_public_pom.xml target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}.pom
-          echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --pinentry-mode loopback \
-            --armor --detach-sign \
-            target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}/databricks-jdbc-thin-${VERSION}.pom
-
-          # Generate checksums for all files
-          cd target/thin-staging/com/databricks/databricks-jdbc-thin/${VERSION}
-          for file in databricks-jdbc-thin-*; do
-            md5sum "$file" | awk '{print $1}' > "${file}.md5"
-            sha1sum "$file" | awk '{print $1}' > "${file}.sha1"
-          done
-          cd $GITHUB_WORKSPACE
-
-          # Create bundle ZIP
-          cd target/thin-staging
-          zip -r ../central-thin-bundle.zip com/
-          cd $GITHUB_WORKSPACE
-
-          echo "Uploading bundle to Maven Central Portal..."
-
-          # Upload to new Maven Central Portal
-          curl -X POST \
-            -u "$MAVEN_CENTRAL_USERNAME:$MAVEN_CENTRAL_PASSWORD" \
-            -F "bundle=@target/central-thin-bundle.zip" \
-            -F "publishingType=AUTOMATIC" \
-            -w "\nHTTP_STATUS:%{http_code}\n" \
-            https://central.sonatype.com/api/v1/publisher/upload
-
-          echo "Thin JAR published successfully!"
+          mvn -Prelease deploy --batch-mode -pl assembly-thin \
+                      -Dnvd.api.key=${{ secrets.NVD_API_KEY }} \
+                      -Dossindex.username=${{ secrets.OSSINDEX_USERNAME }} \
+                      -Dossindex.password=${{ secrets.OSSINDEX_PASSWORD }}
         env:
           GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
           MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
@@ -174,5 +100,4 @@ jobs:
         with:
           tag_name: ${{ steps.get_tag.outputs.tag }}
           files: |
-            target/*-thin.jar
-
+            assembly-thin/target/databricks-jdbc-thin-*.jar