Merge pull request #67 from Tinder/universe_query_generate_hashes

Support Universe Query in Bazel-Diff command

Author: tinder-maxwellelliott

integration/WORKSPACE

@@ -19,32 +19,3 @@ maven_install(
         "https://jcenter.bintray.com/",
     ]
 )
-
-http_archive(
-    name = "io_bazel_rules_docker",
-    sha256 = "1698624e878b0607052ae6131aa216d45ebb63871ec497f26c67455b34119c80",
-    strip_prefix = "rules_docker-0.15.0",
-    urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.15.0/rules_docker-v0.15.0.tar.gz"],
-)
-
-load(
-    "@io_bazel_rules_docker//repositories:repositories.bzl",
-    container_repositories = "repositories",
-)
-container_repositories()
-
-load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps")
-
-container_deps()
-
-load(
-    "@io_bazel_rules_docker//container:container.bzl",
-    "container_pull",
-)
-
-container_pull(
-    name = "openjdk_11_slim",
-    digest = "sha256:28b59dc9a129c75349418e3b75508fd8eef3b33dd7d796079d1d19445d907776",
-    registry = "index.docker.io",
-    repository = "library/openjdk",
-)

integration/integration_test.sh

@@ -10,8 +10,7 @@ modified_filepaths_output="$workspace_path/modified_filepaths.txt"
 starting_hashes_json="$output_dir/starting_hashes.json"
 final_hashes_json="$output_dir/final_hashes.json"
 impacted_targets_path="$output_dir/impacted_targets.txt"
-shared_flags="--config=verbose"
-command_options="--incompatible_restrict_string_escapes=false"
+shared_flags=""
 
 export USE_BAZEL_VERSION=last_downstream_green
 
@@ -22,13 +21,13 @@ containsElement () {
   return 1
 }
 
-$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path $starting_hashes_json -co $command_options
+$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path $starting_hashes_json
 
-$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path -m $modified_filepaths_output $final_hashes_json -co $command_options
+$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path -m $modified_filepaths_output $final_hashes_json
 
 awk '{gsub(/:StringGenerator.java": \"\w+\"/,"modifiedhash");print}' $final_hashes_json > /dev/null
 
-$bazel_path run :bazel-diff $shared_flags -- -sh $starting_hashes_json -fh $final_hashes_json -w $workspace_path -b $bazel_path -o $impacted_targets_path -aq "attr('tags', 'manual', //...)" -co $command_options
+$bazel_path run :bazel-diff $shared_flags -- -sh $starting_hashes_json -fh $final_hashes_json -w $workspace_path -b $bazel_path -o $impacted_targets_path -aq "attr('tags', 'manual', //...)"
 
 IFS=$'\n' read -d '' -r -a impacted_targets < $impacted_targets_path
 target1="//test/java/com/integration:bazel-diff-integration-test-lib"

src/main/java/com/bazel_diff/BazelClient.java

@@ -20,7 +20,7 @@
 
 interface BazelClient {
     List<BazelTarget> queryAllTargets() throws IOException;
-    Set<String> queryForImpactedTargets(Set<String> impactedTargets, String avoidQuery) throws IOException;
+    Set<String> queryForImpactedTargets(Set<String> impactedTargets, String avoidQuery, String universeQuery) throws IOException;
     Set<BazelSourceFileTarget> convertFilepathsToSourceTargets(Set<Path> filepaths) throws IOException, NoSuchAlgorithmException;
     Set<BazelSourceFileTarget> queryAllSourcefileTargets() throws IOException, NoSuchAlgorithmException;
 }
@@ -47,12 +47,12 @@ public List<BazelTarget> queryAllTargets() throws IOException {
     }
 
     @Override
-    public Set<String> queryForImpactedTargets(Set<String> impactedTargets, String avoidQuery) throws IOException {
+    public Set<String> queryForImpactedTargets(Set<String> impactedTargets, String avoidQuery, String universeQuery) throws IOException {
         Set<String> impactedTargetNames = new HashSet<>();
         String targetQuery = impactedTargets.stream()
                                             .map(target -> String.format("'%s'", target))
                                             .collect(Collectors.joining(" + "));
-        String query = query = String.format("rdeps(//... except '//external:all-targets', %s)", targetQuery);
+        String query = query = String.format("rdeps(%s except '//external:all-targets', %s)", universeQuery, targetQuery);
         if (avoidQuery != null) {
             query = String.format("(%s) except (%s)", query, avoidQuery);
         }

src/main/java/com/bazel_diff/TargetHashingClient.java

@@ -12,7 +12,7 @@
 interface TargetHashingClient {
     Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths, Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException;
     Map<String, String> hashAllBazelTargetsAndSourcefiles(Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException;
-    Set<String> getImpactedTargets(Map<String, String> startHashes, Map<String, String> endHashes, String avoidQuery, Boolean hashAllTargets) throws IOException;
+    Set<String> getImpactedTargets(Map<String, String> startHashes, Map<String, String> endHashes, String avoidQuery, String universeQuery, Boolean hashAllTargets) throws IOException;
 }
 
 class TargetHashingClientImpl implements TargetHashingClient {
@@ -41,6 +41,7 @@ public Set<String> getImpactedTargets(
         Map<String, String> startHashes,
         Map<String, String> endHashes,
         String avoidQuery,
+        String universeQuery,
         Boolean hashAllTargets)
     throws IOException {
         Set<String> impactedTargets = new HashSet<>();
@@ -53,7 +54,7 @@ public Set<String> getImpactedTargets(
         if (hashAllTargets != null && hashAllTargets && avoidQuery == null) {
             return impactedTargets;
         }
-        return bazelClient.queryForImpactedTargets(impactedTargets, avoidQuery);
+        return bazelClient.queryForImpactedTargets(impactedTargets, avoidQuery, universeQuery);
     }
 
     private byte[] createDigestForTarget(

src/main/java/com/bazel_diff/main.java

@@ -153,6 +153,15 @@ public Integer call() {
         versionProvider = VersionProvider.class
 )
 class BazelDiff implements Callable<Integer> {
+    @ArgGroup(exclusive = true)
+    Exclusive exclusive;
+    static class Exclusive {
+        @Option(names = {"-a", "--all-sourcefiles"}, description = "Experimental: Hash all sourcefile targets (instead of relying on --modifiedFilepaths), Warning: Performance may degrade from reading all source files")
+        Boolean hashAllSourcefiles;
+
+        @Option(names = {"-u", "--universeQuery"}, description = "The universe query to use when executing `rdeps()` queries, use this to limit the search scope of impacted targets i.e. ignore external targets and such")
+        String universeRdepsQuery;
+    }
 
     @Option(names = {"-w", "--workspacePath"}, description = "Path to Bazel workspace directory.", scope = ScopeType.INHERIT, required = true)
     Path workspacePath;
@@ -169,9 +178,6 @@ class BazelDiff implements Callable<Integer> {
     @Option(names = {"-o", "--output"}, scope = ScopeType.LOCAL, description = "Filepath to write the impacted Bazel targets to, newline separated")
     File outputPath;
 
-    @Option(names = {"-a", "--all-sourcefiles"}, description = "Experimental: Hash all sourcefile targets (instead of relying on --modifiedFilepaths), Warning: Performance may degrade from reading all source files")
-    Boolean hashAllSourcefiles;
-
     @Option(names = {"-aq", "--avoid-query"}, scope = ScopeType.LOCAL, description = "A Bazel query string, any targets that pass this query will be removed from the returned set of targets")
     String avoidQuery;
 
@@ -221,7 +227,16 @@ public Integer call() throws IOException {
         Map<String, String > gsonHash = new HashMap<>();
         Map<String, String> startingHashes = gson.fromJson(startingFileReader, gsonHash.getClass());
         Map<String, String> finalHashes = gson.fromJson(finalFileReader, gsonHash.getClass());
-        Set<String> impactedTargets = hashingClient.getImpactedTargets(startingHashes, finalHashes, avoidQuery, hashAllSourcefiles);
+        Boolean shouldHashAllSourceFiles = false;
+        String universeQuery = "//...";
+        if (exclusive != null) {
+            if (exclusive.hashAllSourcefiles) {
+                shouldHashAllSourceFiles = exclusive.hashAllSourcefiles;
+            } else {
+                universeQuery = exclusive.universeRdepsQuery;
+            }
+        }
+        Set<String> impactedTargets = hashingClient.getImpactedTargets(startingHashes, finalHashes, avoidQuery, universeQuery, shouldHashAllSourceFiles);
         try {
             FileWriter myWriter = new FileWriter(outputPath);
             myWriter.write(impactedTargets.stream().collect(Collectors.joining(System.lineSeparator())));

test/java/com/bazel_diff/TargetHashingClientImplTests.java

@@ -163,7 +163,7 @@ public void hashAllBazelTargets_sourceTargets_modifiedSources_seedFilepaths() th
 
     @Test
     public void getImpactedTargets() throws IOException {
-        when(bazelClientMock.queryForImpactedTargets(anySet(), anyObject())).thenReturn(
+        when(bazelClientMock.queryForImpactedTargets(anySet(), anyObject(), anyObject())).thenReturn(
                 new HashSet<>(Arrays.asList("rule1", "rule3"))
         );
         TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock);
@@ -174,7 +174,7 @@ public void getImpactedTargets() throws IOException {
         hash2.put("rule1", "differentrule1hash");
         hash2.put("rule2", "rule2hash");
         hash2.put("rule3", "rule3hash");
-        Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, null, false);
+        Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, null, null, false);
         Set<String> expectedSet = new HashSet<>();
         expectedSet.add("rule1");
         expectedSet.add("rule3");
@@ -183,7 +183,7 @@ public void getImpactedTargets() throws IOException {
 
     @Test
     public void getImpactedTargets_withAvoidQuery() throws IOException {
-        when(bazelClientMock.queryForImpactedTargets(anySet(), eq("some_query"))).thenReturn(
+        when(bazelClientMock.queryForImpactedTargets(anySet(), eq("some_query"), eq("universe_query"))).thenReturn(
                 new HashSet<>(Arrays.asList("rule1"))
         );
         TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock);
@@ -194,15 +194,15 @@ public void getImpactedTargets_withAvoidQuery() throws IOException {
         hash2.put("rule1", "differentrule1hash");
         hash2.put("rule2", "rule2hash");
         hash2.put("rule3", "rule3hash");
-        Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, "some_query", false);
+        Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, "some_query", "universe_query", false);
         Set<String> expectedSet = new HashSet<>();
         expectedSet.add("rule1");
         assertEquals(expectedSet, impactedTargets);
     }
 
     @Test
     public void getImpactedTargets_withHashAllTargets() throws IOException {
-        when(bazelClientMock.queryForImpactedTargets(anySet(), anyObject())).thenReturn(
+        when(bazelClientMock.queryForImpactedTargets(anySet(), anyObject(), anyObject())).thenReturn(
                 new HashSet<>(Arrays.asList("rule1"))
         );
         TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock);
@@ -213,7 +213,7 @@ public void getImpactedTargets_withHashAllTargets() throws IOException {
         hash2.put("rule1", "differentrule1hash");
         hash2.put("rule2", "rule2hash");
         hash2.put("rule3", "rule3hash");
-        Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, null, true);
+        Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, null, null, true);
         Set<String> expectedSet = new HashSet<>();
         expectedSet.add("rule1");
         expectedSet.add("rule3");
@@ -222,7 +222,7 @@ public void getImpactedTargets_withHashAllTargets() throws IOException {
 
     @Test
     public void getImpactedTargets_withHashAllTargets_withAvoidQuery() throws IOException {
-        when(bazelClientMock.queryForImpactedTargets(anySet(), eq("some_query"))).thenReturn(
+        when(bazelClientMock.queryForImpactedTargets(anySet(), eq("some_query"), anyObject())).thenReturn(
                 new HashSet<>(Arrays.asList("rule1"))
         );
         TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock);
@@ -233,7 +233,7 @@ public void getImpactedTargets_withHashAllTargets_withAvoidQuery() throws IOExce
         hash2.put("rule1", "differentrule1hash");
         hash2.put("rule2", "rule2hash");
         hash2.put("rule3", "rule3hash");
-        Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, "some_query", true);
+        Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, "some_query", null, true);
         Set<String> expectedSet = new HashSet<>();
         expectedSet.add("rule1");
         assertEquals(expectedSet, impactedTargets);