Merge pull request #66 from Tinder/seed_hash_files

tinder-maxwellelliott · web-flow · commit b1d406bf669a · 2021-05-26T13:03:37.000-07:00
Seed Hashfiles
diff --git a/README.md b/README.md
@@ -134,31 +134,39 @@ Writes to the file the modified filepaths between two revisions.
 `generate-hashes` Command
 
 ```terminal
-Usage: bazel-diff generate-hashes [-hV] -b=<bazelPath>
-                                  [-co=<bazelCommandOptions>]
-                                  [-m=<modifiedFilepaths>]
+Usage: bazel-diff generate-hashes [-m=<modifiedFilepaths> | -a] [-hV]
+                                  -b=<bazelPath> [-co=<bazelCommandOptions>]
+                                  [-s=<seedFilepaths>]
                                   [-so=<bazelStartupOptions>]
                                   -w=<workspacePath> <outputPath>
 Writes to a file the SHA256 hashes for each Bazel Target in the provided
 workspace.
-      <outputPath>   The filepath to write the resulting JSON of dictionary
-                       target => SHA-256 values
+      <outputPath>        The filepath to write the resulting JSON of
+                            dictionary target => SHA-256 values
+  -a, --all-sourcefiles   Experimental: Hash all sourcefile targets (instead of
+                            relying on --modifiedFilepaths), Warning:
+                            Performance may degrade from reading all source
+                            files
   -b, --bazelPath=<bazelPath>
-                     Path to Bazel binary
+                          Path to Bazel binary
       -co, --bazelCommandOptions=<bazelCommandOptions>
-                     Additional space separated Bazel command options used when
-                       invoking Bazel
-  -h, --help         Show this help message and exit.
+                          Additional space separated Bazel command options used
+                            when invoking Bazel
+  -h, --help              Show this help message and exit.
   -m, --modifiedFilepaths=<modifiedFilepaths>
-                     The path to a file containing the list of modified
-                       filepaths in the workspace, you can use the
-                       'modified-filepaths' command to get this list
+                          The path to a file containing the list of modified
+                            filepaths in the workspace, you can use the
+                            'modified-filepaths' command to get this list
+  -s, --seed-filepaths=<seedFilepaths>
+                          A text file containing a newline separated list of
+                            filepaths, each of these filepaths will be read and
+                            used as a seed for all targets.
       -so, --bazelStartupOptions=<bazelStartupOptions>
-                     Additional space separated Bazel client startup options
-                       used when invoking Bazel
-  -V, --version      Print version information and exit.
+                          Additional space separated Bazel client startup
+                            options used when invoking Bazel
+  -V, --version           Print version information and exit.
   -w, --workspacePath=<workspacePath>
-                     Path to Bazel workspace directory.
+                          Path to Bazel workspace directory.
 ```
 
 ## Installing
diff --git a/src/main/java/com/bazel_diff/Files.java b/src/main/java/com/bazel_diff/Files.java
@@ -0,0 +1,21 @@
+package com.bazel_diff;
+
+import java.nio.file.Path;
+import java.nio.file.Files;
+import java.io.IOException;
+
+interface FilesClient {
+    byte[] readFile(Path path) throws IOException;
+}
+
+class FilesClientImp implements FilesClient {
+    FilesClientImp() {}
+
+    @Override
+    public byte[] readFile(Path path) throws IOException {
+        if (path.toFile().exists() && path.toFile().canRead()) {
+            return Files.readAllBytes(path);
+        }
+        return new byte[0];
+    }
+}
diff --git a/src/main/java/com/bazel_diff/TargetHashingClient.java b/src/main/java/com/bazel_diff/TargetHashingClient.java
@@ -1,35 +1,39 @@
 package com.bazel_diff;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.*;
 import java.util.stream.Collectors;
+import com.google.common.primitives.Bytes;
 
 interface TargetHashingClient {
-    Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths) throws IOException, NoSuchAlgorithmException;
-    Map<String, String> hashAllBazelTargetsAndSourcefiles() throws IOException, NoSuchAlgorithmException;
+    Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths, Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException;
+    Map<String, String> hashAllBazelTargetsAndSourcefiles(Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException;
     Set<String> getImpactedTargets(Map<String, String> startHashes, Map<String, String> endHashes, String avoidQuery, Boolean hashAllTargets) throws IOException;
 }
 
 class TargetHashingClientImpl implements TargetHashingClient {
     private BazelClient bazelClient;
+    private FilesClient files;
 
-    TargetHashingClientImpl(BazelClient bazelClient) {
+    TargetHashingClientImpl(BazelClient bazelClient, FilesClient files) {
         this.bazelClient = bazelClient;
+        this.files = files;
     }
 
     @Override
-    public Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths) throws IOException, NoSuchAlgorithmException {
+    public Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths, Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException {
         Set<BazelSourceFileTarget> bazelSourcefileTargets = bazelClient.convertFilepathsToSourceTargets(modifiedFilepaths);
-        return hashAllTargets(bazelSourcefileTargets);
+        return hashAllTargets(createSeedForFilepaths(seedFilepaths), bazelSourcefileTargets);
     }
 
     @Override
-    public Map<String, String> hashAllBazelTargetsAndSourcefiles() throws IOException, NoSuchAlgorithmException {
+    public Map<String, String> hashAllBazelTargetsAndSourcefiles(Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException {
         Set<BazelSourceFileTarget> bazelSourcefileTargets = bazelClient.queryAllSourcefileTargets();
-        return hashAllTargets(bazelSourcefileTargets);
+        return hashAllTargets(createSeedForFilepaths(seedFilepaths), bazelSourcefileTargets);
     }
 
     @Override
@@ -56,7 +60,8 @@ private byte[] createDigestForTarget(
             BazelTarget target,
             Map<String, BazelRule> allRulesMap,
             Set<BazelSourceFileTarget> bazelSourcefileTargets,
-            Map<String, byte[]> ruleHashes
+            Map<String, byte[]> ruleHashes,
+            byte[] seedHash
     ) throws NoSuchAlgorithmException {
         BazelRule targetRule = target.getRule();
         if (target.hasSourceFile()) {
@@ -67,24 +72,31 @@ private byte[] createDigestForTarget(
                 if (sourceTargetDigestBytes != null) {
                     digest.update(sourceTargetDigestBytes);
                 }
+                if (seedHash != null) {
+                    digest.update(seedHash);
+                }
                 return digest.digest().clone();
             }
         }
-        return createDigestForRule(targetRule, allRulesMap, ruleHashes, bazelSourcefileTargets);
+        return createDigestForRule(targetRule, allRulesMap, ruleHashes, bazelSourcefileTargets, seedHash);
     }
 
     private byte[] createDigestForRule(
             BazelRule rule,
             Map<String, BazelRule> allRulesMap,
             Map<String, byte[]> ruleHashes,
-            Set<BazelSourceFileTarget> bazelSourcefileTargets
+            Set<BazelSourceFileTarget> bazelSourcefileTargets,
+            byte[] seedHash
     ) throws NoSuchAlgorithmException {
         byte[] existingByteArray = ruleHashes.get(rule.getName());
         if (existingByteArray != null) {
             return existingByteArray;
         }
         MessageDigest digest = MessageDigest.getInstance("SHA-256");
         digest.update(rule.getDigest());
+        if (seedHash != null) {
+            digest.update(seedHash);
+        }
         for (String ruleInput : rule.getRuleInputList()) {
             digest.update(ruleInput.getBytes());
             BazelRule inputRule = allRulesMap.get(ruleInput);
@@ -94,7 +106,8 @@ private byte[] createDigestForRule(
                         inputRule,
                         allRulesMap,
                         ruleHashes,
-                        bazelSourcefileTargets
+                        bazelSourcefileTargets,
+                        seedHash
                 );
                 if (ruleInputHash != null) {
                     digest.update(ruleInputHash);
@@ -108,6 +121,17 @@ private byte[] createDigestForRule(
         return finalHashValue;
     }
 
+    private byte[] createSeedForFilepaths(Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException {
+        if (seedFilepaths == null || seedFilepaths.size() == 0) {
+            return new byte[0];
+        }
+        MessageDigest digest = MessageDigest.getInstance("SHA-256");
+        for (Path path: seedFilepaths) {
+            digest.update(this.files.readFile(path));
+        }
+        return digest.digest().clone();
+    }
+
     private byte[] getDigestForSourceTargetName(
             String sourceTargetName,
             Set<BazelSourceFileTarget> bazelSourcefileTargets
@@ -138,7 +162,7 @@ private String getNameForTarget(BazelTarget target) {
         return null;
     }
 
-    private Map<String, String> hashAllTargets(Set<BazelSourceFileTarget> bazelSourcefileTargets) throws IOException, NoSuchAlgorithmException {
+    private Map<String, String> hashAllTargets(byte[] seedHash, Set<BazelSourceFileTarget> bazelSourcefileTargets) throws IOException, NoSuchAlgorithmException {
         List<BazelTarget> allTargets = bazelClient.queryAllTargets();
         Map<String, String> targetHashes = new HashMap<>();
         Map<String, byte[]> ruleHashes = new HashMap<>();
@@ -159,10 +183,13 @@ private Map<String, String> hashAllTargets(Set<BazelSourceFileTarget> bazelSourc
                     target,
                     allRulesMap,
                     bazelSourcefileTargets,
-                    ruleHashes
+                    ruleHashes,
+                    seedHash
             );
             if (targetDigest != null) {
-                targetHashes.put(targetName, convertByteArrayToString(targetDigest));
+                ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+                outputStream.write(targetDigest);
+                targetHashes.put(targetName, convertByteArrayToString(outputStream.toByteArray()));
             }
         }
         return targetHashes;
diff --git a/src/main/java/com/bazel_diff/main.java b/src/main/java/com/bazel_diff/main.java
@@ -92,17 +92,28 @@ static class Exclusive {
         Boolean hashAllSourcefiles;
     }
 
+    @Option(names = {"-s", "--seed-filepaths"}, description = "A text file containing a newline separated list of filepaths, each of these filepaths will be read and used as a seed for all targets.")
+    File seedFilepaths;
+
     @Parameters(index = "0", description = "The filepath to write the resulting JSON of dictionary target => SHA-256 values")
     File outputPath;
 
     @Override
     public Integer call() {
         GitClient gitClient = new GitClientImpl(parent.workspacePath);
         BazelClient bazelClient = new BazelClientImpl(parent.workspacePath, parent.bazelPath, parent.bazelStartupOptions, parent.bazelCommandOptions, BazelDiff.isVerbose());
-        TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient);
+        TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient, new FilesClientImp());
         try {
             gitClient.ensureAllChangesAreCommitted();
             Set<Path> modifiedFilepathsSet = new HashSet<>();
+            Set<Path> seedFilepathsSet = new HashSet<>();
+            if (seedFilepaths != null) {
+                FileReader fileReader = new FileReader(seedFilepaths);
+                BufferedReader bufferedReader = new BufferedReader(fileReader);
+                seedFilepathsSet = bufferedReader.lines()
+                                                .map( line -> new File(line).toPath())
+                                                .collect(Collectors.toSet());
+            }
             Map<String, String> hashes = new HashMap<>();
             if (exclusive != null) {
                 if (exclusive.modifiedFilepaths != null) {
@@ -112,12 +123,12 @@ public Integer call() {
                             .lines()
                             .map( line -> new File(line).toPath())
                             .collect(Collectors.toSet());
-                    hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet);
+                    hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet, seedFilepathsSet);
                 } else if (exclusive.hashAllSourcefiles) {
-                    hashes = hashingClient.hashAllBazelTargetsAndSourcefiles();
+                    hashes = hashingClient.hashAllBazelTargetsAndSourcefiles(seedFilepathsSet);
                 }
             } else {
-                hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet);
+                hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet, seedFilepathsSet);
             }
             Gson gson = new GsonBuilder().setPrettyPrinting().create();
             FileWriter myWriter = new FileWriter(outputPath);
@@ -182,7 +193,7 @@ public Integer call() throws IOException {
         }
         GitClient gitClient = new GitClientImpl(workspacePath);
         BazelClient bazelClient = new BazelClientImpl(workspacePath, bazelPath, bazelStartupOptions, bazelCommandOptions, BazelDiff.isVerbose());
-        TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient);
+        TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient, new FilesClientImp());
         try {
             gitClient.ensureAllChangesAreCommitted();
         } catch (IOException | InterruptedException e) {
diff --git a/test/java/com/bazel_diff/TargetHashingClientImplTests.java b/test/java/com/bazel_diff/TargetHashingClientImplTests.java
