Skip to content

Commit 0a60a58

Browse files
bz-canvabz-canva
authored
Add --includTargetType and --targetType options (#194)
* Write type to output as well * Add --includeTargetType and --targetType options
1 parent 62df794 commit 0a60a58

File tree

11 files changed

+200
-44
lines changed

11 files changed

+200
-44
lines changed

‎README.md‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ Commands:
8282

8383
```terminal
8484
Usage: bazel-diff generate-hashes [-hkvV] [--[no-]useCquery] [-b=<bazelPath>]
85+
[--[no-]includeTargetType]
8586
[--contentHashPath=<contentHashPath>]
8687
[-s=<seedFilepaths>] -w=<workspacePath>
8788
[-co=<bazelCommandOptions>]...
@@ -92,10 +93,33 @@ Usage: bazel-diff generate-hashes [-hkvV] [--[no-]useCquery] [-b=<bazelPath>]
9293
[-so=<bazelStartupOptions>]... <outputPath>
9394
Writes to a file the SHA256 hashes for each Bazel Target in the provided
9495
workspace.
95-
<outputPath> The filepath to write the resulting JSON of
96-
dictionary target => SHA-256 values. If not
97-
specified, the JSON will be written to STDOUT.
98-
-b, --bazelPath=<bazelPath>
96+
<outputPath> The filepath to write the resulting JSON to.
97+
If not specified, the JSON will be written to STDOUT.
98+
99+
By default the JSON schema is a dictionary of target => SHA-256 values.
100+
Example:
101+
{
102+
"//cli:bazel-diff_deploy.jar": "4ae310f8ad2bc728934e3509b6102ca658e828b9cd668f79990e95c6663f9633"
103+
...
104+
}
105+
106+
If --includeTargetType is specified, the JSON schema will include the target type (SourceFile/Rule/GeneratedFile)
107+
Example:
108+
{
109+
"//cli:src/test/resources/fixture/integration-test-1.zip": "SourceFile#c259eba8539f4c14e4536c61975457c2990e090067893f4a2981e7bb5f4ef4cf",
110+
"//external:android_gmaven_r8": "Rule#795f583449a40814c05e1cc5d833002afed8d12bce5b835579c7f139c2462d61",
111+
"//cli:bazel-diff_deploy.jar": "GeneratedFile#4ae310f8ad2bc728934e3509b6102ca658e828b9cd668f79990e95c6663f9633",
112+
...
113+
}
114+
----[no-]includeTargetType
115+
Whether include target type in the generated JSON or not.
116+
If false, the generate JSON schema is: {"<target>": "<sha256>"}
117+
If true, the generate JSON schema is: {"<target>": "<type>#<sha256>"
118+
-tt, --targetType=<targetType>
119+
The type of targets to filter, available options are SourceFile/Rule/GeneratedFile
120+
Only works if the JSON was generated with `--includeTargetType` enabled.
121+
If not specified, all types of impacted targets will be returned.
122+
-b, --bazelPath=<bazelPath>
99123
Path to Bazel binary. If not specified, the Bazel
100124
binary available in PATH will be used.
101125
-co, --bazelCommandOptions=<bazelCommandOptions>
@@ -157,7 +181,7 @@ See https://github.com/bazelbuild/bazel/issues/17743 for more details.
157181
### What does the SHA256 value of `generate-hashes` represent?
158182

159183
`generate-hashes` is a canonical SHA256 value representing all attributes and inputs into a target. These inputs
160-
are the summation of the of the rule implementation hash, the SHA256 value
184+
are the summation of the rule implementation hash, the SHA256 value
161185
for every attribute of the rule and then the summation of the SHA256 value for
162186
all `rule_inputs` using the same exact algorithm. For source_file inputs the
163187
content of the file are converted into a SHA256 value.
@@ -167,18 +191,24 @@ content of the file are converted into a SHA256 value.
167191
```terminal
168192
Usage: bazel-diff get-impacted-targets [-v] -fh=<finalHashesJSONPath>
169193
-o=<outputPath>
194+
-tt=<targetType>
170195
-sh=<startingHashesJSONPath>
171196
Command-line utility to analyze the state of the bazel build graph
172197
-fh, --finalHashes=<finalHashesJSONPath>
173198
The path to the JSON file of target hashes for the final
174199
revision. Run 'generate-hashes' to get this value.
175-
-o, --output=<outputPath>
200+
-o, --output=<outputPath>
176201
Filepath to write the impacted Bazel targets to, newline
177202
separated
178203
-sh, --startingHashes=<startingHashesJSONPath>
179204
The path to the JSON file of target hashes for the initial
180205
revision. Run 'generate-hashes' to get this value.
181-
-v, --verbose Display query string, missing files and elapsed time
206+
-tt, --targetType=<targetType>
207+
The type of targets to filter, available options are SourceFile/Rule/GeneratedFile
208+
Only works if the JSON was generated with `--includeTargetType` enabled.
209+
If not specified, all types of impacted targets will be returned.
210+
-v, --verbose
211+
Display query string, missing files and elapsed time
182212
```
183213

184214
## Installing

‎cli/src/main/kotlin/com/bazel_diff/cli/GenerateHashesCommand.kt‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,24 @@ class GenerateHashesCommand : Callable<Int> {
8181
)
8282
var useCquery = false
8383

84+
@CommandLine.Option(
85+
names = ["--includeTargetType"],
86+
negatable = true,
87+
description = ["Whether include target type in the generated JSON or not.\n"
88+
+ "If false, the generate JSON schema is: {\"<target>\": \"<sha256>\"}\n"
89+
+ "If true, the generate JSON schema is: {\"<target>\": \"<type>#<sha256>\" }"],
90+
scope = CommandLine.ScopeType.INHERIT
91+
)
92+
var includeTargetType = false
93+
94+
@CommandLine.Option(
95+
names = ["-tt", "--targetType"],
96+
split = ",",
97+
scope = CommandLine.ScopeType.LOCAL,
98+
description = ["The types of targets to filter. Use comma (,) to separate multiple values, e.g. '--targetType=SourceFile,Rule,GeneratedFile'."]
99+
)
100+
var targetType: Set<String>? = null
101+
84102
@CommandLine.Option(
85103
names = ["--cqueryCommandOptions"],
86104
description = ["Additional space separated Bazel command options used when invoking `bazel cquery`. This flag is has no effect if `--useCquery`is false."],
@@ -142,7 +160,7 @@ class GenerateHashesCommand : Callable<Int> {
142160
)
143161
}
144162

145-
return when (GenerateHashesInteractor().execute(seedFilepaths, outputPath, ignoredRuleHashingAttributes)) {
163+
return when (GenerateHashesInteractor().execute(seedFilepaths, outputPath, ignoredRuleHashingAttributes, targetType, includeTargetType)) {
146164
true -> CommandLine.ExitCode.OK
147165
false -> CommandLine.ExitCode.SOFTWARE
148166
}.also { stopKoin() }

‎cli/src/main/kotlin/com/bazel_diff/cli/GetImpactedTargetsCommand.kt‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,14 @@ class GetImpactedTargetsCommand : Callable<Int> {
3838
)
3939
lateinit var finalHashesJSONPath: File
4040

41+
@CommandLine.Option(
42+
names = ["-tt", "--targetType"],
43+
split = ",",
44+
scope = CommandLine.ScopeType.LOCAL,
45+
description = ["The types of targets to filter. Use comma (,) to separate multiple values, e.g. '--targetType=SourceFile,Rule,GeneratedFile'."]
46+
)
47+
var targetType: Set<String>? = null
48+
4149
@CommandLine.Option(
4250
names = ["-o", "--output"],
4351
scope = CommandLine.ScopeType.LOCAL,
@@ -58,8 +66,8 @@ class GetImpactedTargetsCommand : Callable<Int> {
5866

5967
validate()
6068
val deserialiser = DeserialiseHashesInteractor()
61-
val from = deserialiser.execute(startingHashesJSONPath)
62-
val to = deserialiser.execute(finalHashesJSONPath)
69+
val from = deserialiser.execute(startingHashesJSONPath, targetType)
70+
val to = deserialiser.execute(finalHashesJSONPath, targetType)
6371

6472
val impactedTargets = CalculateImpactedTargetsInteractor().execute(from, to)
6573

@@ -70,7 +78,7 @@ class GetImpactedTargetsCommand : Callable<Int> {
7078
}).use { writer ->
7179
impactedTargets.forEach {
7280
writer.write(it)
73-
//Should not be depend on OS
81+
//Should not depend on OS
7482
writer.write("\n")
7583
}
7684
}

‎cli/src/main/kotlin/com/bazel_diff/hash/BuildGraphHasher.kt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
2929
fun hashAllBazelTargetsAndSourcefiles(
3030
seedFilepaths: Set<Path> = emptySet(),
3131
ignoredAttrs: Set<String> = emptySet()
32-
): Map<String, String> {
32+
): Map<String, TargetHash> {
3333
/**
3434
* Bazel will lock parallel queries but this is still allowing us to hash source files while executing a parallel query
3535
*/
@@ -104,7 +104,7 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
104104
sourceDigests: ConcurrentMap<String, ByteArray>,
105105
allTargets: List<BazelTarget>,
106106
ignoredAttrs: Set<String>
107-
): Map<String, String> {
107+
): Map<String, TargetHash> {
108108
val ruleHashes: ConcurrentMap<String, ByteArray> = ConcurrentHashMap()
109109
val targetToRule: MutableMap<String, BazelRule> = HashMap()
110110
traverseGraph(allTargets, targetToRule)
@@ -119,13 +119,13 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
119119
seedHash,
120120
ignoredAttrs
121121
)
122-
Pair(target.name, targetDigest.toHexString())
122+
Pair(target.name, TargetHash(target.javaClass.name.substringAfterLast('$'), targetDigest.toHexString()))
123123
}
124-
.filter { targetEntry: Pair<String, String>? -> targetEntry != null }
124+
.filter { targetEntry: Pair<String, TargetHash>? -> targetEntry != null }
125125
.collect(
126126
Collectors.toMap(
127-
{ obj: Pair<String, String> -> obj.first },
128-
{ obj: Pair<String, String> -> obj.second },
127+
{ obj: Pair<String, TargetHash> -> obj.first },
128+
{ obj: Pair<String, TargetHash> -> obj.second },
129129
)
130130
)
131131
}

‎cli/src/main/kotlin/com/bazel_diff/hash/TargetHash.kt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package com.bazel_diff.hash
2+
3+
data class TargetHash(
4+
val type: String, // Rule/GeneratedFile/SourceFile/...
5+
val hash: String
6+
) {
7+
val hashWithType by lazy {
8+
"${type}#${hash}"
9+
}
10+
11+
fun toJson(includeTargetType: Boolean): String {
12+
return if (includeTargetType) {
13+
hashWithType
14+
} else {
15+
hash
16+
}
17+
}
18+
}

‎cli/src/main/kotlin/com/bazel_diff/interactor/DeserialiseHashesInteractor.kt‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,22 @@ class DeserialiseHashesInteractor : KoinComponent {
1212

1313
/**
1414
* @param file path to file that has been pre-validated
15+
* @param targetTypes the target types to filter. If null, all targets will be returned
1516
*/
16-
fun execute(file: File): Map<String, String> {
17+
fun execute(file: File, targetTypes: Set<String>? = null): Map<String, String> {
1718
val shape = object : TypeToken<Map<String, String>>() {}.type
18-
return gson.fromJson(FileReader(file), shape)
19+
val result: Map<String, String> = gson.fromJson(FileReader(file), shape)
20+
if (targetTypes == null) {
21+
return result.mapValues { it.value.substringAfter("#") }
22+
} else {
23+
val prefixes = targetTypes.map { "${it}#" }.toSet()
24+
return result.filter { entry ->
25+
if (entry.value.contains("#")) {
26+
prefixes.any { entry.value.startsWith(it) }
27+
} else {
28+
throw IllegalStateException("No type info found in ${file}, please re-generate the JSON with --includeTypeTarget!")
29+
}
30+
}.mapValues { it.value.substringAfter("#") }
31+
}
1932
}
2033
}

‎cli/src/main/kotlin/com/bazel_diff/interactor/GenerateHashesInteractor.kt‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ class GenerateHashesInteractor : KoinComponent {
1818
private val logger: Logger by inject()
1919
private val gson: Gson by inject()
2020

21-
fun execute(seedFilepaths: File?, outputPath: File?, ignoredRuleHashingAttributes: Set<String>): Boolean {
21+
fun execute(seedFilepaths: File?, outputPath: File?, ignoredRuleHashingAttributes: Set<String>, targetTypes:Set<String>?, includeTargetType: Boolean = false): Boolean {
2222
return try {
2323
val epoch = Calendar.getInstance().getTimeInMillis()
24-
var seedFilepathsSet: Set<Path> = when {
24+
val seedFilepathsSet: Set<Path> = when {
2525
seedFilepaths != null -> {
2626
BufferedReader(FileReader(seedFilepaths)).use {
2727
it.readLines()
@@ -34,12 +34,18 @@ class GenerateHashesInteractor : KoinComponent {
3434
val hashes = buildGraphHasher.hashAllBazelTargetsAndSourcefiles(
3535
seedFilepathsSet,
3636
ignoredRuleHashingAttributes
37-
)
37+
).let {
38+
if(targetTypes == null) {
39+
it
40+
} else {
41+
it.filter { targetTypes.contains(it.value.type) }
42+
}
43+
}
3844
when (outputPath) {
3945
null -> FileWriter(FileDescriptor.out)
4046
else -> FileWriter(outputPath)
41-
}.use {
42-
it.write(gson.toJson(hashes))
47+
}.use { fileWriter ->
48+
fileWriter.write(gson.toJson(hashes.mapValues { it.value.toJson(includeTargetType) }))
4349
}
4450
val duration = Calendar.getInstance().getTimeInMillis() - epoch;
4551
logger.i { "generate-hashes finished in $duration" }

‎cli/src/test/kotlin/com/bazel_diff/e2e/E2ETest.kt‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,9 @@ class E2ETest {
1616
@get:Rule
1717
val temp: TemporaryFolder = TemporaryFolder()
1818

19-
@Test
20-
fun testE2E() {
19+
private fun CommandLine.execute(args: List<String>) = execute(*args.toTypedArray())
20+
21+
private fun testE2E(extraGenerateHashesArgs: List<String>, extraGetImpactedTargetsArgs: List<String>, expectedResultFile: String) {
2122
val projectA = extractFixtureProject("/fixture/integration-test-1.zip")
2223
val projectB = extractFixtureProject("/fixture/integration-test-2.zip")
2324

@@ -32,24 +33,39 @@ class E2ETest {
3233
val cli = CommandLine(BazelDiff())
3334
//From
3435
cli.execute(
35-
"generate-hashes", "-w", workingDirectoryA.absolutePath, "-b", bazelPath, from.absolutePath
36+
listOf("generate-hashes", "-w", workingDirectoryA.absolutePath, "-b", bazelPath, from.absolutePath) + extraGenerateHashesArgs
3637
)
3738
//To
3839
cli.execute(
39-
"generate-hashes", "-w", workingDirectoryB.absolutePath, "-b", bazelPath, to.absolutePath
40+
listOf("generate-hashes", "-w", workingDirectoryB.absolutePath, "-b", bazelPath, to.absolutePath) + extraGenerateHashesArgs
4041
)
4142
//Impacted targets
4243
cli.execute(
43-
"get-impacted-targets", "-sh", from.absolutePath, "-fh", to.absolutePath, "-o", impactedTargetsOutput.absolutePath
44+
listOf("get-impacted-targets", "-sh", from.absolutePath, "-fh", to.absolutePath, "-o", impactedTargetsOutput.absolutePath) + extraGetImpactedTargetsArgs
4445
)
4546

4647
val actual: Set<String> = impactedTargetsOutput.readLines().filter { it.isNotBlank() }.toSet()
4748
val expected: Set<String> =
48-
javaClass.getResourceAsStream("/fixture/impacted_targets-1-2.txt").use { it.bufferedReader().readLines().filter { it.isNotBlank() }.toSet() }
49+
javaClass.getResourceAsStream(expectedResultFile).use { it.bufferedReader().readLines().filter { it.isNotBlank() }.toSet() }
4950

5051
assertThat(actual).isEqualTo(expected)
5152
}
5253

54+
@Test
55+
fun testE2E() {
56+
testE2E(emptyList(), emptyList(), "/fixture/impacted_targets-1-2.txt")
57+
}
58+
59+
@Test
60+
fun testE2EIncludingTargetType() {
61+
testE2E(listOf("-tt", "Rule,SourceFile"), emptyList(), "/fixture/impacted_targets-1-2-rule-sourcefile.txt")
62+
}
63+
64+
@Test
65+
fun testE2EWithTargetType() {
66+
testE2E(listOf("--includeTargetType"), listOf("-tt", "Rule,SourceFile"), "/fixture/impacted_targets-1-2-rule-sourcefile.txt")
67+
}
68+
5369
@Test
5470
fun testFineGrainedHashExternalRepo() {
5571
// The difference between these two snapshots is simply upgrading the Guava version.

‎cli/src/test/kotlin/com/bazel_diff/hash/BuildGraphHasherTest.kt‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,8 @@ class BuildGraphHasherTest : KoinTest {
7373

7474
val hash = hasher.hashAllBazelTargetsAndSourcefiles()
7575
assertThat(hash).containsOnly(
76-
"rule1" to "2c963f7c06bc1cead7e3b4759e1472383d4469fc3238dc42f8848190887b4775",
77-
"rule2" to "bdc1abd0a07103cea34199a9c0d1020619136ff90fb88dcc3a8f873c811c1fe9",
76+
"rule1" to TargetHash("Rule", "2c963f7c06bc1cead7e3b4759e1472383d4469fc3238dc42f8848190887b4775"),
77+
"rule2" to TargetHash("Rule", "bdc1abd0a07103cea34199a9c0d1020619136ff90fb88dcc3a8f873c811c1fe9"),
7878
)
7979
}
8080

@@ -86,8 +86,8 @@ class BuildGraphHasherTest : KoinTest {
8686
whenever(bazelClientMock.queryAllSourcefileTargets()).thenReturn(emptyList())
8787
val hash = hasher.hashAllBazelTargetsAndSourcefiles(seedFilepaths)
8888
assertThat(hash).containsOnly(
89-
"rule1" to "0404d80eadcc2dbfe9f0d7935086e1115344a06bd76d4e16af0dfd7b4913ee60",
90-
"rule2" to "6fe63fa16340d18176e6d6021972c65413441b72135247179362763508ebddfe",
89+
"rule1" to TargetHash("Rule", "0404d80eadcc2dbfe9f0d7935086e1115344a06bd76d4e16af0dfd7b4913ee60"),
90+
"rule2" to TargetHash("Rule", "6fe63fa16340d18176e6d6021972c65413441b72135247179362763508ebddfe"),
9191
)
9292
}
9393

@@ -103,10 +103,10 @@ class BuildGraphHasherTest : KoinTest {
103103
whenever(bazelClientMock.queryAllSourcefileTargets()).thenReturn(emptyList())
104104
val hash = hasher.hashAllBazelTargetsAndSourcefiles()
105105
assertThat(hash).containsOnly(
106-
"rule1" to "2c963f7c06bc1cead7e3b4759e1472383d4469fc3238dc42f8848190887b4775",
107-
"rule2" to "bdc1abd0a07103cea34199a9c0d1020619136ff90fb88dcc3a8f873c811c1fe9",
108-
"rule3" to "87dd050f1ca0f684f37970092ff6a02677d995718b5a05461706c0f41ffd4915",
109-
"rule4" to "a7bc5d23cd98c4942dc879c649eb9646e38eddd773f9c7996fa0d96048cf63dc",
106+
"rule1" to TargetHash("Rule", "2c963f7c06bc1cead7e3b4759e1472383d4469fc3238dc42f8848190887b4775"),
107+
"rule2" to TargetHash("Rule", "bdc1abd0a07103cea34199a9c0d1020619136ff90fb88dcc3a8f873c811c1fe9"),
108+
"rule3" to TargetHash("Rule", "87dd050f1ca0f684f37970092ff6a02677d995718b5a05461706c0f41ffd4915"),
109+
"rule4" to TargetHash("Rule", "a7bc5d23cd98c4942dc879c649eb9646e38eddd773f9c7996fa0d96048cf63dc"),
110110
)
111111
}
112112

@@ -122,10 +122,10 @@ class BuildGraphHasherTest : KoinTest {
122122
whenever(bazelClientMock.queryAllSourcefileTargets()).thenReturn(emptyList())
123123
val hash = hasher.hashAllBazelTargetsAndSourcefiles()
124124
assertThat(hash).containsOnly(
125-
"rule1" to "2c963f7c06bc1cead7e3b4759e1472383d4469fc3238dc42f8848190887b4775",
126-
"rule2" to "bdc1abd0a07103cea34199a9c0d1020619136ff90fb88dcc3a8f873c811c1fe9",
127-
"rule3" to "ca2f970a5a5a18730d7633cc32b48b1d94679f4ccaea56c4924e1f9913bd9cb5",
128-
"rule4" to "bf15e616e870aaacb02493ea0b8e90c6c750c266fa26375e22b30b78954ee523",
125+
"rule1" to TargetHash("Rule", "2c963f7c06bc1cead7e3b4759e1472383d4469fc3238dc42f8848190887b4775"),
126+
"rule2" to TargetHash("Rule", "bdc1abd0a07103cea34199a9c0d1020619136ff90fb88dcc3a8f873c811c1fe9"),
127+
"rule3" to TargetHash("Rule", "ca2f970a5a5a18730d7633cc32b48b1d94679f4ccaea56c4924e1f9913bd9cb5"),
128+
"rule4" to TargetHash("Rule", "bf15e616e870aaacb02493ea0b8e90c6c750c266fa26375e22b30b78954ee523"),
129129
)
130130
}
131131

0 commit comments

Comments
 (0)