From daf6054fbd4506293726f86c3b4360b67ff5534b Mon Sep 17 00:00:00 2001 From: Eka Winata Date: Wed, 28 May 2025 23:29:51 +0700 Subject: [PATCH 1/2] [improvement] Skip all nested struct subfield to lower SQL executions --- .../maxcompute/schema/SchemaDifferenceUtils.java | 16 +++++++++++----- .../util/SchemaDifferenceUtilsTest.java | 8 +++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/gotocompany/depot/maxcompute/schema/SchemaDifferenceUtils.java b/src/main/java/com/gotocompany/depot/maxcompute/schema/SchemaDifferenceUtils.java index 33aab0b5..d24223d3 100644 --- a/src/main/java/com/gotocompany/depot/maxcompute/schema/SchemaDifferenceUtils.java +++ b/src/main/java/com/gotocompany/depot/maxcompute/schema/SchemaDifferenceUtils.java @@ -53,18 +53,24 @@ private static List getMaxComputeColumnDetailDifference( if (Objects.isNull(oldMetadata)) { //handle new column / struct field changedMetadata.add(entry.getValue()); if (isStructType(entry.getValue().getTypeInfo()) || isStructArrayType(entry.getValue().getTypeInfo())) { - skipStructFields(entry, newMaxComputeColumnDetailIterator); + StructTypeInfo structTypeInfo = isStructType(entry.getValue().getTypeInfo()) ? (StructTypeInfo) entry.getValue().getTypeInfo() : ((StructTypeInfo) ((ArrayTypeInfo) entry.getValue().getTypeInfo()).getElementTypeInfo()); + skipStructFields(structTypeInfo, newMaxComputeColumnDetailIterator); } } } return changedMetadata; } - private static void skipStructFields(Map.Entry entry, Iterator> newMaxComputeColumnDetailIterator) { - StructTypeInfo structTypeInfo = isStructType(entry.getValue().getTypeInfo()) ? (StructTypeInfo) entry.getValue().getTypeInfo() - : ((StructTypeInfo) ((ArrayTypeInfo) entry.getValue().getTypeInfo()).getElementTypeInfo()); + private static void skipStructFields(StructTypeInfo structTypeInfo, Iterator> newMaxComputeColumnDetailIterator) { for (int i = 0; i < structTypeInfo.getFieldCount(); i++) { - newMaxComputeColumnDetailIterator.next(); + TypeInfo currentTypeInfo = structTypeInfo.getFieldTypeInfos().get(i); + if (isPrimitiveType(currentTypeInfo) || isPrimitiveArrayType(currentTypeInfo)) { + newMaxComputeColumnDetailIterator.next(); + } else if (isStructType(currentTypeInfo) || isStructArrayType(currentTypeInfo)) { + newMaxComputeColumnDetailIterator.next(); + StructTypeInfo currentStructTypeInfo = isStructType(currentTypeInfo) ? (StructTypeInfo) currentTypeInfo : ((StructTypeInfo) ((ArrayTypeInfo) currentTypeInfo).getElementTypeInfo()); + skipStructFields(currentStructTypeInfo, newMaxComputeColumnDetailIterator); + } } } diff --git a/src/test/java/com/gotocompany/depot/maxcompute/util/SchemaDifferenceUtilsTest.java b/src/test/java/com/gotocompany/depot/maxcompute/util/SchemaDifferenceUtilsTest.java index c120e3e8..e2b5d4bc 100644 --- a/src/test/java/com/gotocompany/depot/maxcompute/util/SchemaDifferenceUtilsTest.java +++ b/src/test/java/com/gotocompany/depot/maxcompute/util/SchemaDifferenceUtilsTest.java @@ -44,18 +44,16 @@ public void testGetSchemaDifferenceDdl() { .build(); Set expectedMetadataColumns = new HashSet<>(Arrays.asList( "alter table test_schema.test_table add column if not exists `col3`.`f3` array;", + "alter table test_schema.test_table add column if not exists `metadata_2` string;", "alter table test_schema.test_table add column if not exists `col4`.element.`f42` struct<`f421`:string>;", "alter table test_schema.test_table add column if not exists `col5` array>;", "alter table test_schema.test_table add column if not exists `col6` struct<`f61`:string>;", - "alter table test_schema.test_table add column if not exists `metadata_2` string;", - "alter table test_schema.test_table add column if not exists `col7` struct<`order`:struct<`f711`:string,`f712`:int>,`end`:string>;", - "alter table test_schema.test_table add column if not exists `col7`.`order`.`f711` string;", - "alter table test_schema.test_table add column if not exists `col7`.`order`.`f712` int;" + "alter table test_schema.test_table add column if not exists `col7` struct<`order`:struct<`f711`:string,`f712`:int>,`end`:string>;" )); Set actualMetadataColumns = new HashSet<>(SchemaDifferenceUtils.getSchemaDifferenceSql(oldTableSchema, newTableSchema, "test_schema", "test_table")); - assertEquals(actualMetadataColumns.size(), expectedMetadataColumns.size()); + assertEquals(expectedMetadataColumns.size(), actualMetadataColumns.size()); assertTrue(expectedMetadataColumns.containsAll(actualMetadataColumns)); } From d848796f0133b8a15d44a7091acf7c6313f0e968 Mon Sep 17 00:00:00 2001 From: Eka Winata Date: Wed, 28 May 2025 23:30:56 +0700 Subject: [PATCH 2/2] [chore] bump depot version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 41d2682b..f9d8a172 100644 --- a/build.gradle +++ b/build.gradle @@ -22,7 +22,7 @@ plugins { } group 'com.gotocompany' -version '0.10.19' +version '0.10.20' repositories { mavenLocal()