From 7cf4c94cc8d666b1d6151d6b20016bf2b04e16fc Mon Sep 17 00:00:00 2001 From: Drew Dara-Abrams Date: Mon, 4 Aug 2025 13:16:40 -0700 Subject: [PATCH 1/2] feat: Add --stdout option to CLI for JSON output to stdout Enables JSON reports to be optionally output directly to stdout for piping to other programs, rather than being written to file system. Log output is suppressed. Involves changing how output directory is handled, but not intended to change existing default functionality in the CLI or the GUI. closes #1194 --- .../app/gui/GtfsValidatorApp.java | 4 +- .../app/gui/ValidationDisplay.java | 4 +- .../app/gui/GtfsValidatorAppTest.java | 4 +- cli/build.gradle | 4 + .../gtfsvalidator/cli/Arguments.java | 29 ++++- .../gtfsvalidator/cli/ArgumentsTest.java | 51 +++++++-- docs/USAGE.md | 27 +++++ gradle/libs.versions.toml | 3 +- .../JsonReportSummaryGenerator.java | 4 +- .../runner/ValidationRunner.java | 108 +++++++++++++----- .../runner/ValidationRunnerConfig.java | 18 ++- .../model/JsonReportSummaryGeneratorTest.java | 3 +- .../runner/ValidationRunnerTest.java | 4 +- .../web/service/util/ValidationHandler.java | 4 +- .../service/util/ValidationHandlerTest.java | 17 ++- 15 files changed, 222 insertions(+), 62 deletions(-) diff --git a/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.java b/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.java index 7690b36609..ff82bd32b9 100644 --- a/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.java +++ b/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorApp.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.ResourceBundle; import javax.swing.BorderFactory; import javax.swing.Box; @@ -371,6 +372,7 @@ private void runValidation() { private ValidationRunnerConfig buildConfig() throws URISyntaxException { ValidationRunnerConfig.Builder config = ValidationRunnerConfig.builder(); config.setPrettyJson(true); + config.setStdoutOutput(false); String gtfsInput = gtfsInputField.getText(); if (gtfsInput.isBlank()) { @@ -386,7 +388,7 @@ private ValidationRunnerConfig buildConfig() throws URISyntaxException { if (outputDirectory.isBlank()) { throw new IllegalStateException("outputDirectoryField is blank"); } - config.setOutputDirectory(Path.of(outputDirectory)); + config.setOutputDirectory(Optional.of(Path.of(outputDirectory))); Object numThreads = numThreadsSpinner.getValue(); if (numThreads instanceof Integer) { diff --git a/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/ValidationDisplay.java b/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/ValidationDisplay.java index 968e18e9ac..42ea8fae7b 100644 --- a/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/ValidationDisplay.java +++ b/app/gui/src/main/java/org/mobilitydata/gtfsvalidator/app/gui/ValidationDisplay.java @@ -18,9 +18,9 @@ void handleResult(ValidationRunnerConfig config, ValidationRunner.Status status) handleError(); } - Path reportPath = config.htmlReportPath(); + Path reportPath = config.htmlReportPath().orElse(null); if (status == ValidationRunner.Status.SYSTEM_ERRORS) { - reportPath = config.systemErrorsReportPath(); + reportPath = config.systemErrorsReportPath().orElse(null); } try { diff --git a/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorAppTest.java b/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorAppTest.java index 6a2e710ab9..251cea5995 100644 --- a/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorAppTest.java +++ b/app/gui/src/test/java/org/mobilitydata/gtfsvalidator/app/gui/GtfsValidatorAppTest.java @@ -66,7 +66,7 @@ public void testValidationConfig() throws URISyntaxException { ValidationRunnerConfig config = configCaptor.getValue(); assertThat(config.gtfsSource()).isEqualTo(new URI("http://transit/gtfs.zip")); - assertThat(config.outputDirectory()).isEqualTo(Path.of("/path/to/output")); + assertThat(config.outputDirectory()).hasValue(Path.of("/path/to/output")); assertThat(config.numThreads()).isEqualTo(1); assertThat(config.countryCode().isUnknown()).isTrue(); } @@ -84,7 +84,7 @@ public void testValidationConfigWithAdvancedOptions() throws URISyntaxException ValidationRunnerConfig config = configCaptor.getValue(); assertThat(config.gtfsSource()).isEqualTo(new URI("http://transit/gtfs.zip")); - assertThat(config.outputDirectory()).isEqualTo(Path.of("/path/to/output")); + assertThat(config.outputDirectory()).hasValue(Path.of("/path/to/output")); assertThat(config.numThreads()).isEqualTo(5); assertThat(config.countryCode().getCountryCode()).isEqualTo("US"); } diff --git a/cli/build.gradle b/cli/build.gradle index 600b870f07..1364db54d2 100644 --- a/cli/build.gradle +++ b/cli/build.gradle @@ -38,6 +38,9 @@ shadowJar { // from minimization. exclude(project(':main')) exclude(dependency(libs.httpclient5.get().toString())) + // Keep SLF4J implementation to avoid warnings + // Note that SLF4J comes from some transitive dependencies + exclude(dependency('org.slf4j:slf4j-jdk14')) } // Change the JAR name from 'main' to 'gtfs-validator' archiveBaseName = rootProject.name @@ -55,6 +58,7 @@ dependencies { implementation libs.flogger implementation libs.flogger.system.backend implementation libs.guava + implementation libs.slf4j.jul testImplementation libs.junit testImplementation libs.truth diff --git a/cli/src/main/java/org/mobilitydata/gtfsvalidator/cli/Arguments.java b/cli/src/main/java/org/mobilitydata/gtfsvalidator/cli/Arguments.java index 9715110783..1dba56aafb 100644 --- a/cli/src/main/java/org/mobilitydata/gtfsvalidator/cli/Arguments.java +++ b/cli/src/main/java/org/mobilitydata/gtfsvalidator/cli/Arguments.java @@ -23,6 +23,7 @@ import java.nio.file.Path; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.Optional; import org.mobilitydata.gtfsvalidator.input.CountryCode; import org.mobilitydata.gtfsvalidator.runner.ValidationRunnerConfig; @@ -38,8 +39,7 @@ public class Arguments { @Parameter( names = {"-o", "--output_base"}, - description = "Base directory to store the outputs", - required = true) + description = "Base directory to store the outputs (required if not using --stdout)") private String outputBase; @Parameter( @@ -110,6 +110,11 @@ public class Arguments { description = "Skips check for new validator version") private boolean skipValidatorUpdate = false; + @Parameter( + names = {"--stdout"}, + description = "Output JSON report to stdout instead of writing to files (conflicts with -o)") + private boolean stdoutOutput = false; + ValidationRunnerConfig toConfig() throws URISyntaxException { ValidationRunnerConfig.Builder builder = ValidationRunnerConfig.builder(); if (input != null) { @@ -121,7 +126,10 @@ ValidationRunnerConfig toConfig() throws URISyntaxException { } } if (outputBase != null) { - builder.setOutputDirectory(Path.of(outputBase)); + builder.setOutputDirectory(Optional.of(Path.of(outputBase))); + } else if (stdoutOutput) { + // When using stdout, no output directory is needed + builder.setOutputDirectory(Optional.empty()); } if (countryCode != null) { builder.setCountryCode(CountryCode.forStringOrUnknown(countryCode)); @@ -141,6 +149,7 @@ ValidationRunnerConfig toConfig() throws URISyntaxException { builder.setNumThreads(numThreads); builder.setPrettyJson(pretty); builder.setSkipValidatorUpdate(skipValidatorUpdate); + builder.setStdoutOutput(stdoutOutput); return builder.build(); } @@ -160,6 +169,10 @@ public boolean getExportNoticeSchema() { return exportNoticeSchema; } + public boolean getStdoutOutput() { + return stdoutOutput; + } + /** * @return true if CLI parameter combination is legal, otherwise return false */ @@ -185,6 +198,16 @@ public boolean validate() { return false; } + if (stdoutOutput && outputBase != null) { + logger.atSevere().log("Cannot use --stdout with --output_base. Use one or the other."); + return false; + } + + if (outputBase == null && !stdoutOutput) { + logger.atSevere().log("Must provide either --output_base or --stdout"); + return false; + } + return true; } diff --git a/cli/src/test/java/org/mobilitydata/gtfsvalidator/cli/ArgumentsTest.java b/cli/src/test/java/org/mobilitydata/gtfsvalidator/cli/ArgumentsTest.java index 4cba280185..c968559d7b 100644 --- a/cli/src/test/java/org/mobilitydata/gtfsvalidator/cli/ArgumentsTest.java +++ b/cli/src/test/java/org/mobilitydata/gtfsvalidator/cli/ArgumentsTest.java @@ -18,10 +18,10 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth8.assertThat; -import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import com.beust.jcommander.JCommander; -import com.beust.jcommander.ParameterException; import java.io.File; import java.net.URI; import java.net.URISyntaxException; @@ -45,7 +45,7 @@ public void shortNameShouldInitializeArguments() throws URISyntaxException { new JCommander(underTest).parse(commandLineArgumentAsStringArray); ValidationRunnerConfig config = underTest.toConfig(); assertThat(config.gtfsSource()).isEqualTo(toFileUri("/tmp/gtfs.zip")); - assertThat((Object) config.outputDirectory()).isEqualTo(Path.of("/tmp/output")); + assertThat(config.outputDirectory()).hasValue(Path.of("/tmp/output")); assertThat(config.countryCode()).isEqualTo(CountryCode.forStringOrUnknown("au")); assertThat(config.numThreads()).isEqualTo(4); assertThat(config.validationReportFileName()).matches("report.json"); @@ -72,7 +72,7 @@ public void shortNameShouldInitializeArguments_url() throws URISyntaxException { new JCommander(underTest).parse(commandLineArgumentAsStringArray); ValidationRunnerConfig config = underTest.toConfig(); assertThat(config.gtfsSource()).isEqualTo(new URI("http://host/gtfs.zip")); - assertThat((Object) config.outputDirectory()).isEqualTo(Path.of("/tmp/output")); + assertThat(config.outputDirectory()).hasValue(Path.of("/tmp/output")); assertThat(config.countryCode()).isEqualTo(CountryCode.forStringOrUnknown("au")); assertThat(config.numThreads()).isEqualTo(4); assertThat(config.storageDirectory()).hasValue(Path.of("/tmp/storage")); @@ -95,7 +95,7 @@ public void longNameShouldInitializeArguments() throws URISyntaxException { new JCommander(underTest).parse(commandLineArgumentAsStringArray); ValidationRunnerConfig config = underTest.toConfig(); assertThat(config.gtfsSource()).isEqualTo(toFileUri("/tmp/gtfs.zip")); - assertThat((Object) config.outputDirectory()).isEqualTo(Path.of("/tmp/output")); + assertThat(config.outputDirectory()).hasValue(Path.of("/tmp/output")); assertThat(config.countryCode()).isEqualTo(CountryCode.forStringOrUnknown("ca")); assertThat(config.numThreads()).isEqualTo(4); assertThat(config.validationReportFileName()).matches("report.json"); @@ -131,7 +131,7 @@ public void longNameShouldInitializeArguments_url() throws URISyntaxException { new JCommander(underTest).parse(commandLineArgumentAsStringArray); ValidationRunnerConfig config = underTest.toConfig(); assertThat(config.gtfsSource()).isEqualTo(new URI("http://host/gtfs.zip")); - assertThat((Object) config.outputDirectory()).isEqualTo(Path.of("/tmp/output")); + assertThat(config.outputDirectory()).hasValue(Path.of("/tmp/output")); assertThat(config.countryCode()).isEqualTo(CountryCode.forStringOrUnknown("ca")); assertThat(config.numThreads()).isEqualTo(4); assertThat(config.storageDirectory()).hasValue(Path.of("/tmp/storage")); @@ -152,7 +152,7 @@ public void numThreadsShouldHaveDefaultValueIfNotProvided() throws URISyntaxExce new JCommander(underTest).parse(commandLineArgumentAsStringArray); ValidationRunnerConfig config = underTest.toConfig(); assertThat(config.gtfsSource()).isEqualTo(toFileUri("/tmp/gtfs.zip")); - assertThat((Object) config.outputDirectory()).isEqualTo(Path.of("/tmp/output")); + assertThat(config.outputDirectory()).hasValue(Path.of("/tmp/output")); assertThat(config.countryCode()).isEqualTo(CountryCode.forStringOrUnknown("ca")); assertThat(config.numThreads()).isEqualTo(1); } @@ -182,7 +182,7 @@ public void noUrlNoInput_long_isNotValid() { @Test public void noArguments_isNotValid() { - assertThrows(ParameterException.class, () -> validateArguments(new String[] {})); + assertThat(validateArguments(new String[] {})).isFalse(); } @Test @@ -259,4 +259,39 @@ public void exportNoticesSchema_schemaAndValidation() { assertThat(args.getExportNoticeSchema()).isTrue(); assertThat(args.abortAfterNoticeSchemaExport()).isFalse(); } + + @Test + public void testStdoutOutput() { + String[] commandLineArgumentAsStringArray = {"--input", "test.zip", "--stdout"}; + + Arguments args = new Arguments(); + new JCommander(args).parse(commandLineArgumentAsStringArray); + + assertTrue(args.validate()); + assertTrue(args.getStdoutOutput()); + } + + @Test + public void testStdoutOutputWithOutputBaseConflict() { + String[] commandLineArgumentAsStringArray = { + "--input", "test.zip", + "--output_base", "/tmp/output", + "--stdout" + }; + + Arguments args = new Arguments(); + new JCommander(args).parse(commandLineArgumentAsStringArray); + + assertFalse(args.validate()); + } + + @Test + public void testStdoutOutputWithoutInput() { + String[] commandLineArgumentAsStringArray = {"--stdout"}; + + Arguments args = new Arguments(); + new JCommander(args).parse(commandLineArgumentAsStringArray); + + assertFalse(args.validate()); + } } diff --git a/docs/USAGE.md b/docs/USAGE.md index 0f334cf34c..5301117400 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -26,6 +26,7 @@ | `-e` | `--system_errors_report_name` | Optional | Name of the system errors report (including `.json` extension). | | `-n` | `--export_notices_schema` | Optional | Export notice schema as a json file. | | `-p` | `--pretty` | Optional | Pretty JSON validation report. If specified, the JSON validation report will be printed using JSON Pretty print. This does not impact data parsing. | +| `--stdout` | `--stdout` | Optional | Output JSON report to stdout instead of writing to files. Use with `-i` or `-u` but not with `-o`. Enables piping to tools like `jq`. | | `-d` | `--date` | Optional | The date used to validate the feed for time-based rules, e.g feed_expiration_30_days, in ISO_LOCAL_DATE format like '2001-01-30'. By default, the current date is used. | | `-svu` | `--skip_validator_update` | Optional | Skip GTFS version validation update check. If specified, the GTFS version validation will be skipped. By default, the GTFS version validation will be performed. | @@ -61,6 +62,32 @@ java -jar gtfs-validator-v2.0.jar -u https://url/to/dataset.zip -o relative/outp 1. Validate the GTFS data and output the results to the directory located at `relative/output/path`. Validation results are exported to JSON by default. Please note that since downloading will take time, we recommend validating repeatedly on a local file. +## via stdout output (for scripting and piping) + +The `--stdout` option outputs JSON directly to stdout instead of writing files, making it ideal for scripting and piping to other tools. + +### Basic stdout usage +``` +java -jar gtfs-validator-v2.0.jar -i relative/path/to/dataset.zip --stdout +``` + +### Pipe to jq for processing +``` +java -jar gtfs-validator-v2.0.jar -i relative/path/to/dataset.zip --stdout | jq '.summary.validationTimeSeconds' +``` + +### Pretty JSON output to stdout +``` +java -jar gtfs-validator-v2.0.jar -i relative/path/to/dataset.zip --stdout --pretty +``` + +### URL-based input with stdout +``` +java -jar gtfs-validator-v2.0.jar -u https://url/to/dataset.zip --stdout +``` + +⚠️ Note that `--stdout` cannot be used with `-o` or `--output_base`. Use one or the other. + ## via GitHub Actions - Run the validator on any gtfs archive available on a public url 1. [Fork this repository](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e51f9d22a2..a6c08321e1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -70,4 +70,5 @@ jetbrains-annotations = { module = "org.jetbrains:annotations", version.ref = "j aspectjrt = { module = "org.aspectj:aspectjrt", version.ref = "aspectjrt" } aspectjrt-weaver = { module = "org.aspectj:aspectjweaver", version.ref = "aspectjrtweaver" } findbugs = { module = "com.google.code.findbugs:jsr305", version.ref = "findbugs" } -jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jacksonDatabind" } \ No newline at end of file +jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jacksonDatabind" } +slf4j-jul = { module = "org.slf4j:slf4j-jdk14", version = "1.7.25" } \ No newline at end of file diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportSummaryGenerator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportSummaryGenerator.java index a2abc8b17c..c33fc9bc2f 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportSummaryGenerator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/reportsummary/JsonReportSummaryGenerator.java @@ -31,7 +31,9 @@ public JsonReportSummaryGenerator( date, config != null ? config.gtfsSource().toString() : null, config != null ? config.numThreads() : 0, - config != null ? config.outputDirectory().toString() : null, + config != null && config.outputDirectory().isPresent() + ? config.outputDirectory().get().toString() + : null, config != null ? config.systemErrorsReportFileName() : null, config != null ? config.validationReportFileName() : null, config != null ? config.htmlReportFileName() : null, diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java index f1a67d668a..aa96097820 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java @@ -26,6 +26,7 @@ import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; import java.time.ZonedDateTime; @@ -75,6 +76,11 @@ public ValidationRunner(VersionResolver versionResolver) { @MemoryMonitor public Status run(ValidationRunnerConfig config) { + // Suppress logging when using stdout mode to avoid interfering with JSON output + if (config.stdoutOutput()) { + // Set logging level to SEVERE to minimize output + java.util.logging.Logger.getLogger("").setLevel(java.util.logging.Level.SEVERE); + } MemoryUsageRegister.getInstance().clearRegistry(); // Registering the memory metrics manually to avoid multiple entries due to concurrent calls // and exclude from the metric the generation of the reports. @@ -151,7 +157,7 @@ public Status run(ValidationRunnerConfig config) { // Output exportReport(feedMetadata, noticeContainer, config, versionInfo); - printSummary(feedMetadata, feedContainer, feedLoader); + printSummary(feedMetadata, feedContainer, feedLoader, config); return Status.SUCCESS; } @@ -162,7 +168,14 @@ public Status run(ValidationRunnerConfig config) { * @param feedContainer the {@code GtfsFeedContainer} */ public static void printSummary( - FeedMetadata feedMetadata, GtfsFeedContainer feedContainer, GtfsFeedLoader loader) { + FeedMetadata feedMetadata, + GtfsFeedContainer feedContainer, + GtfsFeedLoader loader, + ValidationRunnerConfig config) { + // Skip summary output when using stdout mode to avoid interfering with JSON output + if (config.stdoutOutput()) { + return; + } final long endNanos = System.nanoTime(); var skippedValidators = loader.getSkippedValidators(); var multiFileValidatorsWithParsingErrors = @@ -296,45 +309,78 @@ public static void exportReport( NoticeContainer noticeContainer, ValidationRunnerConfig config, VersionInfo versionInfo) { - if (!Files.exists(config.outputDirectory())) { - try { - Files.createDirectories(config.outputDirectory()); - } catch (IOException ex) { - logger.atSevere().withCause(ex).log( - "Error creating output directory: %s", config.outputDirectory()); - } - } + ZonedDateTime now = ZonedDateTime.now(); String date = now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssXXX")); - boolean is_different_date = !now.toLocalDate().equals(config.dateForValidation()); - Gson gson = createGson(config.prettyJson()); - HtmlReportGenerator htmlGenerator = new HtmlReportGenerator(); JsonReportGenerator jsonGenerator = new JsonReportGenerator(); + + // Generate JSON report + JsonReport jsonReport = null; try { - JsonReport jsonReport = + jsonReport = jsonGenerator.generateReport(feedMetadata, noticeContainer, config, versionInfo, date); - Files.write( - config.outputDirectory().resolve(config.validationReportFileName()), - gson.toJson(jsonReport).getBytes(StandardCharsets.UTF_8)); } catch (Exception ex) { logger.atSevere().withCause(ex).log("Error creating JSON report"); + return; } - try { - htmlGenerator.generateReport( - feedMetadata, - noticeContainer, - config, - versionInfo, - config.outputDirectory().resolve(config.htmlReportFileName()), - date, - is_different_date); - Files.write( - config.outputDirectory().resolve(config.systemErrorsReportFileName()), - gson.toJson(noticeContainer.exportSystemErrors()).getBytes(StandardCharsets.UTF_8)); - } catch (IOException e) { - logger.atSevere().withCause(e).log("Cannot store report files"); + if (config.stdoutOutput()) { + // Output JSON to stdout + try { + System.out.println(gson.toJson(jsonReport)); + } catch (Exception ex) { + logger.atSevere().withCause(ex).log("Error creating JSON report"); + } + + // System errors to stderr + try { + System.err.println(gson.toJson(noticeContainer.exportSystemErrors())); + } catch (Exception ex) { + logger.atSevere().withCause(ex).log("Error creating system errors report"); + } + return; + } + + // Existing file-based output + if (config.outputDirectory().isPresent()) { + Path outputDir = config.outputDirectory().get(); + if (!Files.exists(outputDir)) { + try { + Files.createDirectories(outputDir); + } catch (IOException ex) { + logger.atSevere().withCause(ex).log("Error creating output directory: %s", outputDir); + } + } + } + boolean is_different_date = !now.toLocalDate().equals(config.dateForValidation()); + + HtmlReportGenerator htmlGenerator = new HtmlReportGenerator(); + if (config.outputDirectory().isPresent()) { + Path outputDir = config.outputDirectory().get(); + try { + Files.write( + outputDir.resolve(config.validationReportFileName()), + gson.toJson(jsonReport).getBytes(StandardCharsets.UTF_8)); + } catch (Exception ex) { + logger.atSevere().withCause(ex).log("Error creating JSON report"); + } + + try { + htmlGenerator.generateReport( + feedMetadata, + noticeContainer, + config, + versionInfo, + outputDir.resolve(config.htmlReportFileName()), + date, + is_different_date); + Files.write( + outputDir.resolve(config.systemErrorsReportFileName()), + gson.toJson(noticeContainer.exportSystemErrors()).getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + logger.atSevere().withCause(e).log("Cannot store report files"); + } } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerConfig.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerConfig.java index 804d6ab165..89c970c839 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerConfig.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerConfig.java @@ -29,7 +29,8 @@ public abstract class ValidationRunnerConfig { public abstract URI gtfsSource(); // The directory where all validation reports will be written. - public abstract Path outputDirectory(); + // Optional when using stdout mode. + public abstract Optional outputDirectory(); // An optional storage directory to be used when downloading a GTFS feed // from an external URL. @@ -39,14 +40,14 @@ public abstract class ValidationRunnerConfig { public abstract String htmlReportFileName(); - public Path htmlReportPath() { - return outputDirectory().resolve(htmlReportFileName()); + public Optional htmlReportPath() { + return outputDirectory().map(dir -> dir.resolve(htmlReportFileName())); } public abstract String systemErrorsReportFileName(); - public Path systemErrorsReportPath() { - return outputDirectory().resolve(systemErrorsReportFileName()); + public Optional systemErrorsReportPath() { + return outputDirectory().map(dir -> dir.resolve(systemErrorsReportFileName())); } // Determines the number of parallel threads of execution used during @@ -66,6 +67,9 @@ public Path systemErrorsReportPath() { // If true, the validator will not check for a new validator version public abstract boolean skipValidatorUpdate(); + // If true, output JSON report to stdout instead of writing to files + public abstract boolean stdoutOutput(); + public static Builder builder() { // Set reasonable defaults where appropriate. return new AutoValue_ValidationRunnerConfig.Builder() @@ -83,7 +87,7 @@ public static Builder builder() { public abstract static class Builder { public abstract Builder setGtfsSource(URI gtfsSource); - public abstract Builder setOutputDirectory(Path outputDirectory); + public abstract Builder setOutputDirectory(Optional outputDirectory); public abstract Builder setStorageDirectory(Path storageDirectory); @@ -103,6 +107,8 @@ public abstract static class Builder { public abstract Builder setSkipValidatorUpdate(boolean skipValidatorUpdate); + public abstract Builder setStdoutOutput(boolean stdoutOutput); + public abstract ValidationRunnerConfig build(); } } diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/reportsummary/model/JsonReportSummaryGeneratorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/reportsummary/model/JsonReportSummaryGeneratorTest.java index 273389747f..20df68d24b 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/reportsummary/model/JsonReportSummaryGeneratorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/reportsummary/model/JsonReportSummaryGeneratorTest.java @@ -40,12 +40,13 @@ private static ValidationRunnerConfig generateValidationRunnerConfig() throws Ex builder.setCountryCode(CountryCode.forStringOrUnknown("GB")); builder.setGtfsSource(new URI("some_dataset_filename")); builder.setHtmlReportFileName("some_html_filename"); - builder.setOutputDirectory(Path.of("some_output_directory")); + builder.setOutputDirectory(Optional.of(Path.of("some_output_directory"))); builder.setNumThreads(1); builder.setPrettyJson(true); builder.setSystemErrorsReportFileName("some_error_filename"); builder.setValidationReportFileName("some_report_filename"); builder.setDateForValidation(LocalDate.parse("2020-01-02")); + builder.setStdoutOutput(false); return builder.build(); } diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerTest.java index 22730c2f4b..6c1d29574e 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerTest.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; +import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -17,9 +18,10 @@ public class ValidationRunnerTest { private static ValidationRunnerConfig buildConfig(String gtfsDirectory) { ValidationRunnerConfig.Builder config = ValidationRunnerConfig.builder(); config.setGtfsSource(Path.of(gtfsDirectory).toUri()); - config.setOutputDirectory(Path.of("")); + config.setOutputDirectory(Optional.of(Path.of(""))); config.setNumThreads(1); config.setCountryCode(CountryCode.forStringOrUnknown("")); + config.setStdoutOutput(false); return config.build(); } diff --git a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/ValidationHandler.java b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/ValidationHandler.java index aae34de976..f7fd90774e 100644 --- a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/ValidationHandler.java +++ b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/ValidationHandler.java @@ -2,6 +2,7 @@ import java.io.File; import java.nio.file.Path; +import java.util.Optional; import lombok.NonNull; import lombok.RequiredArgsConstructor; import org.mobilitydata.gtfsvalidator.input.CountryCode; @@ -32,7 +33,8 @@ public void validateFeed(@NonNull File feedFile, @NonNull Path outputPath, Strin var configBuilder = ValidationRunnerConfig.builder() .setGtfsSource(feedFile.toURI()) - .setOutputDirectory(outputPath); + .setOutputDirectory(Optional.of(outputPath)) + .setStdoutOutput(false); if (!countryCode.isEmpty()) { var country = CountryCode.forStringOrUnknown(countryCode); logger.debug("setting country code: {}", country.getCountryCode()); diff --git a/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/util/ValidationHandlerTest.java b/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/util/ValidationHandlerTest.java index 138f7b7220..8c42845835 100644 --- a/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/util/ValidationHandlerTest.java +++ b/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/util/ValidationHandlerTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; @@ -40,7 +41,9 @@ public void testValidationHandlerRunnerSuccessNoCountryCode() throws Exception { verify(runner, times(1)).run(configCaptor.capture()); var config = configCaptor.getValue(); assert config.gtfsSource().equals(feedFileURI); - assert config.outputDirectory().equals(mockOutputPath); + assertTrue( + config.outputDirectory().isPresent() + && mockOutputPath.equals(config.outputDirectory().get())); assert config.countryCode().equals(CountryCode.forStringOrUnknown(countryCode)); } @@ -60,7 +63,9 @@ public void testValidationHandlerRunnerSuccessWithCountryCode() throws Exception verify(runner, times(1)).run(configCaptor.capture()); var config = configCaptor.getValue(); assert config.gtfsSource().equals(feedFileURI); - assert config.outputDirectory().equals(mockOutputPath); + assertTrue( + config.outputDirectory().isPresent() + && mockOutputPath.equals(config.outputDirectory().get())); assert config.countryCode().equals(CountryCode.forStringOrUnknown(countryCode)); } @@ -85,7 +90,9 @@ public void testValidationHandlerRunnerExceptionStatus() throws Exception { verify(runner, times(1)).run(configCaptor.capture()); var config = configCaptor.getValue(); assert config.gtfsSource().equals(feedFileURI); - assert config.outputDirectory().equals(mockOutputPath); + assertTrue( + config.outputDirectory().isPresent() + && mockOutputPath.equals(config.outputDirectory().get())); assert config.countryCode().equals(CountryCode.forStringOrUnknown(countryCode)); } @@ -112,7 +119,9 @@ public void testValidationHandlerRunnerSystemErrorsStatus() throws Exception { verify(runner, times(1)).run(configCaptor.capture()); var config = configCaptor.getValue(); assert config.gtfsSource().equals(feedFileURI); - assert config.outputDirectory().equals(mockOutputPath); + assertTrue( + config.outputDirectory().isPresent() + && mockOutputPath.equals(config.outputDirectory().get())); assert config.countryCode().equals(CountryCode.forStringOrUnknown(countryCode)); } } From 53a7d69b5e18d0686714f4fb166098a7e31f79ed Mon Sep 17 00:00:00 2001 From: Drew Dara-Abrams Date: Mon, 4 Aug 2025 13:53:09 -0700 Subject: [PATCH 2/2] don't emit to stderror --- docs/USAGE.md | 2 ++ .../mobilitydata/gtfsvalidator/runner/ValidationRunner.java | 6 ------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/USAGE.md b/docs/USAGE.md index 5301117400..b549ceb8d3 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -88,6 +88,8 @@ java -jar gtfs-validator-v2.0.jar -u https://url/to/dataset.zip --stdout ⚠️ Note that `--stdout` cannot be used with `-o` or `--output_base`. Use one or the other. +⚠️ When using `--stdout`, all system errors and logging output are suppressed to ensure clean JSON output. Only severe-level log messages (hard crashes) will appear on stderr. + ## via GitHub Actions - Run the validator on any gtfs archive available on a public url 1. [Fork this repository](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java index aa96097820..1473ead661 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java @@ -333,12 +333,6 @@ public static void exportReport( logger.atSevere().withCause(ex).log("Error creating JSON report"); } - // System errors to stderr - try { - System.err.println(gson.toJson(noticeContainer.exportSystemErrors())); - } catch (Exception ex) { - logger.atSevere().withCause(ex).log("Error creating system errors report"); - } return; }