From 5814203bcb28df65f3f34876b273342e67136553 Mon Sep 17 00:00:00 2001 From: Vladislav Sidorovich Date: Thu, 20 Mar 2025 14:34:21 +0100 Subject: [PATCH 1/3] add test template to verify assessment flagUsage --- .../dumper/AssessmentFlagUsageTest.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java new file mode 100644 index 000000000..4d51ac914 --- /dev/null +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java @@ -0,0 +1,30 @@ +package com.google.edwmigration.dumper.application.dumper; + +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableCollection; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; +import com.google.edwmigration.dumper.application.dumper.connector.Connector; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AssessmentFlagUsageTest { + private static final Logger logger = LoggerFactory.getLogger(AssessmentFlagUsageTest.class); + + @Test + public void assessmentFlagIsExplicitlySpecified() throws Exception { + ImmutableCollection connectors = ConnectorRepository.getInstance().getAllConnectors(); + for (Connector connector : connectors) { + RespectsArgumentAssessment assessment = connector.getClass() + .getDeclaredAnnotation(RespectsArgumentAssessment.class); + if (assessment != null) { + assertThrows( + Exception.class, + () -> connector.validate(new ConnectorArguments())); + } else { + logger.warn("Should --assessment flag be required for the connector: {} ?", connector.getName()); + } + } + } +} From 063b66301f735e6a638cde1ecb756edf5e118f86 Mon Sep 17 00:00:00 2001 From: Vladislav Sidorovich Date: Thu, 20 Mar 2025 14:43:22 +0100 Subject: [PATCH 2/3] use annotation util to cover parent classes --- .../dumper/application/dumper/AssessmentFlagUsageTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java index 4d51ac914..1909f1495 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java @@ -8,6 +8,7 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.AnnotationUtils; public class AssessmentFlagUsageTest { private static final Logger logger = LoggerFactory.getLogger(AssessmentFlagUsageTest.class); @@ -16,8 +17,8 @@ public class AssessmentFlagUsageTest { public void assessmentFlagIsExplicitlySpecified() throws Exception { ImmutableCollection connectors = ConnectorRepository.getInstance().getAllConnectors(); for (Connector connector : connectors) { - RespectsArgumentAssessment assessment = connector.getClass() - .getDeclaredAnnotation(RespectsArgumentAssessment.class); + RespectsArgumentAssessment assessment = AnnotationUtils.findAnnotation(connector.getClass(), + RespectsArgumentAssessment.class); if (assessment != null) { assertThrows( Exception.class, From bbfb9d14b22bb5d962a3f0c83eb9bd02a9517ccc Mon Sep 17 00:00:00 2001 From: Vladislav Sidorovich Date: Mon, 15 Sep 2025 14:25:08 +0200 Subject: [PATCH 3/3] add AvoidArgumentAssessment and mark all the connectors --- .../annotations/AvoidArgumentAssessment.java | 32 +++++++++ .../dumper/annotations/AvoidInput.java | 37 ++++++++++ .../dumper/annotations/AvoidInputs.java | 30 ++++++++ .../dumper/annotations/RespectsInput.java | 12 ++-- .../bigquery/BigQueryLogsConnector.java | 6 +- .../bigquery/BigQueryMetadataConnector.java | 2 + .../manager/ClouderaManagerConnector.java | 2 + .../connector/generic/GenericConnector.java | 2 + .../greenplum/GreenplumMetadataConnector.java | 2 + .../hadoop/HadoopMetadataConnector.java | 2 + .../hdfs/HdfsExtractionConnector.java | 2 + .../connector/hive/HiveMetadataConnector.java | 2 + .../mysql/MysqlMetadataConnector.java | 2 + .../netezza/NetezzaMetadataConnector.java | 9 +-- .../connector/oracle/OracleLogsConnector.java | 2 + .../oracle/OracleMetadataConnector.java | 2 + .../oracle/OracleStatsConnector.java | 2 + .../PostgresqlMetadataConnector.java | 2 + .../connector/ranger/RangerConnector.java | 2 + .../redshift/RedshiftLogsConnector.java | 6 +- .../redshift/RedshiftMetadataConnector.java | 2 + .../sqlserver/SqlServerMetadataConnector.java | 9 +-- .../teradata/Teradata14LogsConnector.java | 2 + .../vertica/VerticaLogsConnector.java | 2 + .../vertica/VerticaMetadataConnector.java | 2 + .../dumper/AssessmentFlagUsageTest.java | 68 +++++++++++++++---- .../dumper/connector/test/TestConnector.java | 2 + 27 files changed, 204 insertions(+), 41 deletions(-) create mode 100644 dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidArgumentAssessment.java create mode 100644 dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidInput.java create mode 100644 dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidInputs.java diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidArgumentAssessment.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidArgumentAssessment.java new file mode 100644 index 000000000..66c0712ac --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidArgumentAssessment.java @@ -0,0 +1,32 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.annotations; + +import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import java.lang.annotation.*; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@AvoidInput( + arg = ConnectorArguments.OPT_ASSESSMENT, + description = AvoidArgumentAssessment.DESCRIPTION) +public @interface AvoidArgumentAssessment { + + String DESCRIPTION = "The argument must not be used with this connector."; +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidInput.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidInput.java new file mode 100644 index 000000000..5bc23f39d --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidInput.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.annotations; + +import java.lang.annotation.*; +import javax.annotation.Nonnull; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@Repeatable(AvoidInputs.class) +public @interface AvoidInput { + + @Nonnull + String arg() default ""; + + @Nonnull + String description(); + + @Nonnull + String defaultValue() default ""; +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidInputs.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidInputs.java new file mode 100644 index 000000000..a1af55f34 --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/AvoidInputs.java @@ -0,0 +1,30 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper.annotations; + +import java.lang.annotation.*; +import javax.annotation.Nonnull; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +public @interface AvoidInputs { + + @Nonnull + AvoidInput[] value(); +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/RespectsInput.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/RespectsInput.java index 28378c3e4..c66afbac2 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/RespectsInput.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/RespectsInput.java @@ -34,19 +34,19 @@ public @interface RespectsInput { @Nonnull - public String arg() default ""; + String arg() default ""; @Nonnull - public String env() default ""; + String env() default ""; @Nonnull - public String description(); + String description(); @Nonnull - public String defaultValue() default ""; + String defaultValue() default ""; - public int order() default Integer.MAX_VALUE; + int order() default Integer.MAX_VALUE; @Nonnull - public String required() default ""; + String required() default ""; } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/bigquery/BigQueryLogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/bigquery/BigQueryLogsConnector.java index 467f3fc27..9c0be7efb 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/bigquery/BigQueryLogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/bigquery/BigQueryLogsConnector.java @@ -31,10 +31,7 @@ import com.google.cloud.bigquery.TableId; import com.google.common.collect.ImmutableList; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogDays; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogEnd; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogStart; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput; +import com.google.edwmigration.dumper.application.dumper.annotations.*; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.LogsConnector; import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval; @@ -69,6 +66,7 @@ @RespectsArgumentQueryLogDays @RespectsArgumentQueryLogStart @RespectsArgumentQueryLogEnd +@AvoidArgumentAssessment public class BigQueryLogsConnector extends AbstractBigQueryConnector implements LogsConnector, BigQueryLogsDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/bigquery/BigQueryMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/bigquery/BigQueryMetadataConnector.java index 548b31098..e84d463dd 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/bigquery/BigQueryMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/bigquery/BigQueryMetadataConnector.java @@ -37,6 +37,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.AvoidArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.MetadataConnector; @@ -77,6 +78,7 @@ order = 2000, arg = ConnectorArguments.OPT_SCHEMA, description = "The list of datasets to dump, separated by commas.") +@AvoidArgumentAssessment public class BigQueryMetadataConnector extends AbstractBigQueryConnector implements BigQueryMetadataDumpFormat, MetadataConnector { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java index 2c6535b1f..261620af6 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java @@ -23,6 +23,7 @@ import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput; import com.google.edwmigration.dumper.application.dumper.connector.AbstractConnector; import com.google.edwmigration.dumper.application.dumper.connector.Connector; @@ -64,6 +65,7 @@ arg = ConnectorArguments.OPT_CLUSTER, description = "The name of Cloudera's cluster.", required = "Only if you need to dump data for a single Cloudera cluster") +@RespectsArgumentAssessment public class ClouderaManagerConnector extends AbstractConnector { private static final String FORMAT_NAME = "cloudera-manager.dump.zip"; diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/generic/GenericConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/generic/GenericConnector.java index cdcc6950f..86d0844ae 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/generic/GenericConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/generic/GenericConnector.java @@ -19,6 +19,7 @@ import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; +import com.google.edwmigration.dumper.application.dumper.annotations.AvoidArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.connector.AbstractJdbcConnector; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.LogsConnector; @@ -48,6 +49,7 @@ * generates multiple queries and is like 'logs' mode. */ @AutoService({Connector.class, LogsConnector.class}) +@AvoidArgumentAssessment public class GenericConnector extends AbstractJdbcConnector implements LogsConnector { private static final Logger logger = LoggerFactory.getLogger(GenericConnector.class); diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/greenplum/GreenplumMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/greenplum/GreenplumMetadataConnector.java index 361b8cd42..14e1b6008 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/greenplum/GreenplumMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/greenplum/GreenplumMetadataConnector.java @@ -18,6 +18,7 @@ import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.AvoidArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentDatabaseForConnection; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput; import com.google.edwmigration.dumper.application.dumper.connector.Connector; @@ -39,6 +40,7 @@ required = ConnectorArguments.OPT_REQUIRED_IF_NOT_URL, defaultValue = "" + GreenplumMetadataConnector.OPT_PORT_DEFAULT) @RespectsArgumentDatabaseForConnection +@AvoidArgumentAssessment public class GreenplumMetadataConnector extends AbstractGreenplumConnector implements MetadataConnector, GreenplumMetadataDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/HadoopMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/HadoopMetadataConnector.java index ba630caf0..a50c1a880 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/HadoopMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hadoop/HadoopMetadataConnector.java @@ -24,6 +24,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.ConnectorProperty; import com.google.edwmigration.dumper.application.dumper.connector.MetadataConnector; @@ -41,6 +42,7 @@ @AutoService({Connector.class, MetadataConnector.class}) @Description("Dumps metadata from the Hadoop cluster via bash commands.") +@RespectsArgumentAssessment public class HadoopMetadataConnector implements MetadataConnector { @VisibleForTesting diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hdfs/HdfsExtractionConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hdfs/HdfsExtractionConnector.java index 2c73a0a31..8340bf49c 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hdfs/HdfsExtractionConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hdfs/HdfsExtractionConnector.java @@ -18,6 +18,7 @@ import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput; import com.google.edwmigration.dumper.application.dumper.connector.AbstractConnector; import com.google.edwmigration.dumper.application.dumper.connector.Connector; @@ -48,6 +49,7 @@ description = "The size of the thread pool to use when extracting hdfs filesystem.") @AutoService({Connector.class}) @Description("Dumps files and directories from the HDFS.") +@RespectsArgumentAssessment public class HdfsExtractionConnector extends AbstractConnector implements HdfsExtractionDumpFormat { public HdfsExtractionConnector() { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hive/HiveMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hive/HiveMetadataConnector.java index e213b3411..a49fcce62 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hive/HiveMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/hive/HiveMetadataConnector.java @@ -31,6 +31,7 @@ import com.google.common.io.ByteSink; import com.google.common.net.PercentEscaper; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentDatabasePredicate; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.MetadataConnector; @@ -73,6 +74,7 @@ @RespectsArgumentDatabasePredicate @AutoService({Connector.class, MetadataConnector.class}) @Description("Dumps metadata from the Hive metastore via Thrift.") +@RespectsArgumentAssessment public class HiveMetadataConnector extends AbstractHiveConnector implements HiveMetadataDumpFormat, MetadataConnector { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/mysql/MysqlMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/mysql/MysqlMetadataConnector.java index 0236b1b5b..c832a7bd3 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/mysql/MysqlMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/mysql/MysqlMetadataConnector.java @@ -18,6 +18,7 @@ import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.MetadataConnector; import com.google.edwmigration.dumper.application.dumper.task.DumpMetadataTask; @@ -38,6 +39,7 @@ */ @AutoService({Connector.class, MetadataConnector.class}) @Description("Dumps metadata from MySQL.") +@RespectsArgumentAssessment public class MysqlMetadataConnector extends AbstractMysqlConnector implements MetadataConnector, MysqlMetadataDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/netezza/NetezzaMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/netezza/NetezzaMetadataConnector.java index 9537a5d0d..c9d9cecf0 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/netezza/NetezzaMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/netezza/NetezzaMetadataConnector.java @@ -21,13 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentDatabasePredicate; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentDriverRequired; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentHostUnlessUrl; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentJDBCUri; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentPassword; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentUser; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput; +import com.google.edwmigration.dumper.application.dumper.annotations.*; import com.google.edwmigration.dumper.application.dumper.connector.AbstractJdbcConnector; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.MetadataConnector; @@ -85,6 +79,7 @@ @RespectsArgumentPassword @RespectsArgumentJDBCUri @RespectsArgumentDatabasePredicate +@RespectsArgumentAssessment public class NetezzaMetadataConnector extends AbstractJdbcConnector implements MetadataConnector, NetezzaMetadataDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleLogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleLogsConnector.java index 45bfca061..f98811149 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleLogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleLogsConnector.java @@ -18,6 +18,7 @@ import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.AvoidArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.LogsConnector; import com.google.edwmigration.dumper.application.dumper.task.DumpMetadataTask; @@ -31,6 +32,7 @@ @AutoService({Connector.class, LogsConnector.class}) @Description("Dumps query logs from Oracle") +@AvoidArgumentAssessment public class OracleLogsConnector extends AbstractOracleConnector implements LogsConnector, OracleLogsDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleMetadataConnector.java index 104063389..a613f60da 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleMetadataConnector.java @@ -21,6 +21,7 @@ import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.MetadataConnector; import com.google.edwmigration.dumper.application.dumper.connector.oracle.task.GroupTask; @@ -39,6 +40,7 @@ @AutoService({Connector.class, MetadataConnector.class}) @Description("Dumps metadata from Oracle") +@RespectsArgumentAssessment public class OracleMetadataConnector extends AbstractOracleConnector implements MetadataConnector, OracleMetadataDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleStatsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleStatsConnector.java index de179d195..4e2dd75ca 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleStatsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/oracle/OracleStatsConnector.java @@ -22,6 +22,7 @@ import com.google.common.collect.Range; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.task.Task; import com.google.edwmigration.dumper.plugin.ext.jdk.annotation.Description; @@ -32,6 +33,7 @@ @AutoService(Connector.class) @Description("Dumps aggregated statistics from Oracle") @ParametersAreNonnullByDefault +@RespectsArgumentAssessment public class OracleStatsConnector extends AbstractOracleConnector { static final Duration DEFAULT_DURATION = ofDays(30); diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/postgresql/PostgresqlMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/postgresql/PostgresqlMetadataConnector.java index 3844ca3d3..8339440a1 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/postgresql/PostgresqlMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/postgresql/PostgresqlMetadataConnector.java @@ -18,6 +18,7 @@ import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentDatabaseForConnection; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput; import com.google.edwmigration.dumper.application.dumper.connector.Connector; @@ -40,6 +41,7 @@ required = ConnectorArguments.OPT_REQUIRED_IF_NOT_URL, defaultValue = "" + PostgresqlMetadataConnector.OPT_PORT_DEFAULT) @RespectsArgumentDatabaseForConnection +@RespectsArgumentAssessment public class PostgresqlMetadataConnector extends AbstractPostgresqlConnector implements MetadataConnector, PostgresqlMetadataDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/ranger/RangerConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/ranger/RangerConnector.java index a17e177be..7226e2594 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/ranger/RangerConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/ranger/RangerConnector.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.AvoidArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput; import com.google.edwmigration.dumper.application.dumper.connector.AbstractConnector; import com.google.edwmigration.dumper.application.dumper.connector.Connector; @@ -61,6 +62,7 @@ defaultValue = ConnectorArguments.OPT_RANGER_PORT_DEFAULT) @AutoService({Connector.class}) @Description("Dumps services and policies from Apache Ranger.") +@AvoidArgumentAssessment public class RangerConnector extends AbstractConnector { @SuppressWarnings("UnusedVariable") diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/redshift/RedshiftLogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/redshift/RedshiftLogsConnector.java index 9546c042c..85ead13f9 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/redshift/RedshiftLogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/redshift/RedshiftLogsConnector.java @@ -19,10 +19,7 @@ import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogDays; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogEnd; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogStart; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput; +import com.google.edwmigration.dumper.application.dumper.annotations.*; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.LogsConnector; import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval; @@ -59,6 +56,7 @@ @RespectsArgumentQueryLogDays @RespectsArgumentQueryLogStart @RespectsArgumentQueryLogEnd +@AvoidArgumentAssessment public class RedshiftLogsConnector extends AbstractRedshiftConnector implements LogsConnector, RedshiftLogsDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/redshift/RedshiftMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/redshift/RedshiftMetadataConnector.java index 26322b816..9943965ac 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/redshift/RedshiftMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/redshift/RedshiftMetadataConnector.java @@ -19,6 +19,7 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.MetadataConnector; @@ -42,6 +43,7 @@ description = "The port of the server.", required = ConnectorArguments.OPT_REQUIRED_IF_NOT_URL, defaultValue = RedshiftUrlUtil.OPT_PORT_DEFAULT) +@RespectsArgumentAssessment public class RedshiftMetadataConnector extends AbstractRedshiftConnector implements MetadataConnector, RedshiftMetadataDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/sqlserver/SqlServerMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/sqlserver/SqlServerMetadataConnector.java index 5c1422ce4..db5ed4ddd 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/sqlserver/SqlServerMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/sqlserver/SqlServerMetadataConnector.java @@ -18,13 +18,7 @@ import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentDatabaseForConnection; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentDriver; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentHostUnlessUrl; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentJDBCUri; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentPassword; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentUser; -import com.google.edwmigration.dumper.application.dumper.annotations.RespectsInput; +import com.google.edwmigration.dumper.application.dumper.annotations.*; import com.google.edwmigration.dumper.application.dumper.connector.AbstractJdbcConnector; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.MetadataConnector; @@ -67,6 +61,7 @@ @RespectsArgumentDriver @RespectsArgumentDatabaseForConnection @RespectsArgumentJDBCUri +@RespectsArgumentAssessment public class SqlServerMetadataConnector extends AbstractJdbcConnector implements MetadataConnector, SqlServerMetadataDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java index a3c0c73a5..291ef26f5 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java @@ -23,6 +23,7 @@ import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogDays; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogEnd; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogStart; @@ -62,6 +63,7 @@ @RespectsArgumentQueryLogDays @RespectsArgumentQueryLogStart @RespectsArgumentQueryLogEnd +@RespectsArgumentAssessment public class Teradata14LogsConnector extends AbstractTeradataConnector implements LogsConnector, TeradataLogsDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/vertica/VerticaLogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/vertica/VerticaLogsConnector.java index 3e31b62ef..8be60d287 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/vertica/VerticaLogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/vertica/VerticaLogsConnector.java @@ -18,6 +18,7 @@ import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.AvoidArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.LogsConnector; import com.google.edwmigration.dumper.application.dumper.task.DumpMetadataTask; @@ -32,6 +33,7 @@ @AutoService({Connector.class, LogsConnector.class}) @Description("Dumps logs from Vertica.") +@AvoidArgumentAssessment public class VerticaLogsConnector extends AbstractVerticaConnector implements LogsConnector, VerticaLogsDumpFormat { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/vertica/VerticaMetadataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/vertica/VerticaMetadataConnector.java index e1dd983ab..2422d5b2a 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/vertica/VerticaMetadataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/vertica/VerticaMetadataConnector.java @@ -18,6 +18,7 @@ import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.MetadataConnector; import com.google.edwmigration.dumper.application.dumper.task.DumpMetadataTask; @@ -31,6 +32,7 @@ /** */ @AutoService({Connector.class, MetadataConnector.class}) @Description("Dumps metadata from Vertica.") +@RespectsArgumentAssessment public class VerticaMetadataConnector extends AbstractVerticaConnector implements MetadataConnector, VerticaMetadataDumpFormat { diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java index 1909f1495..6c50a1bc6 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/AssessmentFlagUsageTest.java @@ -1,31 +1,73 @@ +/* + * Copyright 2022-2025 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.edwmigration.dumper.application.dumper; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import com.google.common.collect.ImmutableCollection; +import com.google.edwmigration.dumper.application.dumper.annotations.AvoidArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.connector.Connector; +import java.lang.annotation.Annotation; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.core.annotation.AnnotationUtils; public class AssessmentFlagUsageTest { - private static final Logger logger = LoggerFactory.getLogger(AssessmentFlagUsageTest.class); @Test - public void assessmentFlagIsExplicitlySpecified() throws Exception { - ImmutableCollection connectors = ConnectorRepository.getInstance().getAllConnectors(); + public void allTheConnectorsExplicitlyChoseAssessmentFlag() { + ImmutableCollection connectors = allConnectors(); for (Connector connector : connectors) { - RespectsArgumentAssessment assessment = AnnotationUtils.findAnnotation(connector.getClass(), - RespectsArgumentAssessment.class); - if (assessment != null) { - assertThrows( - Exception.class, - () -> connector.validate(new ConnectorArguments())); - } else { - logger.warn("Should --assessment flag be required for the connector: {} ?", connector.getName()); + RespectsArgumentAssessment respect = + findAnnotation(connector, RespectsArgumentAssessment.class); + AvoidArgumentAssessment avoid = findAnnotation(connector, AvoidArgumentAssessment.class); + + String assertMessage = + "The connector " + + connector + + " must use one of " + + RespectsArgumentAssessment.class.getSimpleName() + + " or " + + AvoidArgumentAssessment.class.getSimpleName(); + assertTrue(assertMessage, respect != null ^ avoid != null); + } + } + + @Test + public void assessmentFlagIsExplicitlySpecifiedAndUsed() throws Exception { + ImmutableCollection connectors = allConnectors(); + for (Connector connector : connectors) { + RespectsArgumentAssessment assessment = + findAnnotation(connector, RespectsArgumentAssessment.class); + if (assessment == null) { + continue; } + + assertThrows(Exception.class, () -> connector.validate(new ConnectorArguments())); } } + + private static A findAnnotation( + Connector connector, Class annotationType) { + return AnnotationUtils.findAnnotation(connector.getClass(), annotationType); + } + + private static ImmutableCollection allConnectors() { + return ConnectorRepository.getInstance().getAllConnectors(); + } } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/test/TestConnector.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/test/TestConnector.java index 62ef675ec..64edeaea8 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/test/TestConnector.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/connector/test/TestConnector.java @@ -18,6 +18,7 @@ import com.google.auto.service.AutoService; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; +import com.google.edwmigration.dumper.application.dumper.annotations.AvoidArgumentAssessment; import com.google.edwmigration.dumper.application.dumper.connector.AbstractConnector; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.ConnectorProperty; @@ -28,6 +29,7 @@ import javax.annotation.Nonnull; @AutoService({Connector.class, MetadataConnector.class}) +@AvoidArgumentAssessment public class TestConnector extends AbstractConnector implements MetadataConnector { private static final Handle DUMMY_HANDLE = () -> {};