diff --git a/.gitignore b/.gitignore index d7f340fc9b51..293e46585aa7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.claude/*local.json /_output /openshift.local.* /third-party diff --git a/go.mod b/go.mod index 66936173a773..d01716725f48 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/onsi/ginkgo/v2 v2.23.3 github.com/onsi/gomega v1.37.0 github.com/opencontainers/go-digest v1.0.0 - github.com/openshift-eng/openshift-tests-extension v0.0.0-20251113163031-356b66aa5c24 + github.com/openshift-eng/openshift-tests-extension v0.0.0-20251218142942-7ecc8801b9df github.com/openshift-kni/commatrix v0.0.5-0.20251111204857-e5a931eff73f github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7 github.com/openshift/apiserver-library-go v0.0.0-20251015164739-79d04067059d diff --git a/go.sum b/go.sum index f952a06e27ae..fbe1ecd9555f 100644 --- a/go.sum +++ b/go.sum @@ -822,8 +822,8 @@ github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.1 h1:nHFvthhM0qY8/m+vfhJylliSshm8G1jJ2jDMcgULaH8= github.com/opencontainers/selinux v1.11.1/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= -github.com/openshift-eng/openshift-tests-extension v0.0.0-20251113163031-356b66aa5c24 h1:bwmjtFaipakIwAyZxnDLgtkLY1Nf1nK9lRCmADvHirE= -github.com/openshift-eng/openshift-tests-extension v0.0.0-20251113163031-356b66aa5c24/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20251218142942-7ecc8801b9df h1:/KiCxPFpkZN4HErfAX5tyhn6G3ziPFbkGswHVAZKY5Q= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20251218142942-7ecc8801b9df/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M= github.com/openshift-kni/commatrix v0.0.5-0.20251111204857-e5a931eff73f h1:E72Zoc+JImPehBrXkgaCbIDbSFuItvyX6RCaZ0FQE5k= github.com/openshift-kni/commatrix v0.0.5-0.20251111204857-e5a931eff73f/go.mod h1:cDVdp0eda7EHE6tLuSeo4IqPWdAX/KJK+ogBirIGtsI= github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7 h1:Ot2fbEEPmF3WlPQkyEW/bUCV38GMugH/UmZvxpWceNc= diff --git a/pkg/test/extensions/html_test.go b/pkg/test/extensions/html_test.go new file mode 100644 index 000000000000..c8fbe8e7be66 --- /dev/null +++ b/pkg/test/extensions/html_test.go @@ -0,0 +1,62 @@ +package extensions + +import ( + "testing" + + "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestViewerHTMLTemplateIsRenderable(t *testing.T) { + // This test ensures the viewer.html template can be parsed by Go's template engine. + // It catches issues like unescaped {{ }} sequences in comments or other content + // that would cause the template to fail to parse. + + testData := []byte(`[{"name":"test","result":"passed"}]`) + suiteName := "test-suite" + + html, err := extensiontests.RenderResultsHTML(testData, suiteName) + require.NoError(t, err, "viewer.html template should be parseable") + assert.Contains(t, string(html), "test-suite", "rendered HTML should contain suite name") + assert.Contains(t, string(html), `"name":"test"`, "rendered HTML should contain test data") +} + +func TestToHTMLModes(t *testing.T) { + // Test that both HTML output modes work correctly + results := ExtensionTestResults{ + &ExtensionTestResult{ + ExtensionTestResult: &extensiontests.ExtensionTestResult{ + Name: "passing-test", + Result: extensiontests.ResultPassed, + Output: "some output", + }, + }, + &ExtensionTestResult{ + ExtensionTestResult: &extensiontests.ExtensionTestResult{ + Name: "failing-test", + Result: extensiontests.ResultFailed, + Output: "failure output", + Error: "error message", + }, + }, + } + + t.Run("summary mode", func(t *testing.T) { + html, err := results.ToHTML("test-suite", HTMLOutputSummary) + require.NoError(t, err) + assert.NotEmpty(t, html) + // Summary mode should still contain the test names + assert.Contains(t, string(html), "passing-test") + assert.Contains(t, string(html), "failing-test") + }) + + t.Run("everything mode", func(t *testing.T) { + html, err := results.ToHTML("test-suite", HTMLOutputEverything) + require.NoError(t, err) + assert.NotEmpty(t, html) + // Everything mode should contain all output + assert.Contains(t, string(html), "some output") + assert.Contains(t, string(html), "failure output") + }) +} diff --git a/pkg/test/extensions/types.go b/pkg/test/extensions/types.go index 06636b9f6475..52fd2e9f56d7 100644 --- a/pkg/test/extensions/types.go +++ b/pkg/test/extensions/types.go @@ -1,6 +1,7 @@ package extensions import ( + "encoding/json" "fmt" "strings" @@ -95,6 +96,49 @@ type ExtensionTestResult struct { Source Source `json:"source"` } +// HTMLOutputMode controls what content is included in the HTML output. +type HTMLOutputMode int + +const ( + // HTMLOutputSummary elides output/error/details for passed tests to reduce file size. + HTMLOutputSummary HTMLOutputMode = iota + // HTMLOutputEverything includes all output/error/details for all tests. + HTMLOutputEverything +) + +// ToHTML converts the extension test results to an HTML representation. +// It marshals origin's results (which include SourceImage/SourceBinary) directly +// and uses RenderResultsHTML to preserve those fields in the HTML output. +func (results ExtensionTestResults) ToHTML(suiteName string, mode HTMLOutputMode) ([]byte, error) { + jsonData, err := json.Marshal(results) + if err != nil { + return nil, fmt.Errorf("failed to marshal results: %w", err) + } + + switch mode { + case HTMLOutputSummary: + var copiedResults ExtensionTestResults + if err := json.Unmarshal(jsonData, &copiedResults); err != nil { + return nil, fmt.Errorf("failed to unmarshal results: %w", err) + } + for _, r := range copiedResults { + if r != nil && r.ExtensionTestResult != nil && r.Result == extensiontests.ResultPassed { + r.Error = "" + r.Output = "" + r.Details = nil + } + } + jsonData, err = json.Marshal(copiedResults) + if err != nil { + return nil, fmt.Errorf("failed to marshal results: %w", err) + } + case HTMLOutputEverything: + // Include all output as-is + } + + return extensiontests.RenderResultsHTML(jsonData, suiteName) +} + // EnvironmentFlagName enumerates each possible EnvironmentFlag's name to be passed to the external binary type EnvironmentFlagName string diff --git a/pkg/test/ginkgo/cmd_runsuite.go b/pkg/test/ginkgo/cmd_runsuite.go index 6b8bc2e3b7df..ae793dba5c87 100644 --- a/pkg/test/ginkgo/cmd_runsuite.go +++ b/pkg/test/ginkgo/cmd_runsuite.go @@ -677,7 +677,7 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc fmt.Fprintf(o.Out, "error: Unable to write e2e JUnit xml results: %v", err) } - if err := writeExtensionTestResults(tests, o.JUnitDir, "extension_test_result_e2e", timeSuffix, o.ErrOut); err != nil { + if err := writeExtensionTestResults(tests, o.JUnitDir, "extension_test_result_e2e", timeSuffix, suite.Name, o.ErrOut); err != nil { fmt.Fprintf(o.Out, "error: Unable to write e2e Extension Test Result JSON results: %v", err) } @@ -950,7 +950,7 @@ func writeRunSuiteOptions(seed int64, totalNodes, workerNodes, parallelism int, } } -func writeExtensionTestResults(tests []*testCase, dir, filePrefix, fileSuffix string, out io.Writer) error { +func writeExtensionTestResults(tests []*testCase, dir, filePrefix, fileSuffix, suiteName string, out io.Writer) error { // Ensure the directory exists err := os.MkdirAll(dir, 0755) if err != nil { @@ -989,6 +989,34 @@ func writeExtensionTestResults(tests []*testCase, dir, filePrefix, fileSuffix st return err } + // Generate HTML output (summary - elides passed test outputs) + summaryData, err := results.ToHTML(suiteName, extensions.HTMLOutputSummary) + if err != nil { + fmt.Fprintf(out, "Failed to generate summary HTML: %v\n", err) + return err + } + + summaryPath := filepath.Join(dir, fmt.Sprintf("%s_%s-summary.html", filePrefix, fileSuffix)) + fmt.Fprintf(out, "Writing extension test results HTML to %s\n", summaryPath) + if err := os.WriteFile(summaryPath, summaryData, 0644); err != nil { + fmt.Fprintf(out, "Failed to write HTML file %s: %v\n", summaryPath, err) + return err + } + + // Generate HTML output (everything - includes all outputs) + everythingData, err := results.ToHTML(suiteName, extensions.HTMLOutputEverything) + if err != nil { + fmt.Fprintf(out, "Failed to generate everything HTML: %v\n", err) + return err + } + + everythingPath := filepath.Join(dir, fmt.Sprintf("%s_%s-everything.html", filePrefix, fileSuffix)) + fmt.Fprintf(out, "Writing extension test results HTML to %s\n", everythingPath) + if err := os.WriteFile(everythingPath, everythingData, 0644); err != nil { + fmt.Fprintf(out, "Failed to write HTML file %s: %v\n", everythingPath, err) + return err + } + return nil } diff --git a/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdrun/runsuite.go b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdrun/runsuite.go index d81d07cb2b15..998fd70409e7 100644 --- a/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdrun/runsuite.go +++ b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdrun/runsuite.go @@ -2,9 +2,11 @@ package cmdrun import ( "context" + "encoding/json" "fmt" "os" "os/signal" + "path/filepath" "syscall" "time" @@ -22,11 +24,13 @@ func NewRunSuiteCommand(registry *extension.Registry) *cobra.Command { outputFlags *flags.OutputFlags concurrencyFlags *flags.ConcurrencyFlags junitPath string + htmlPath string }{ componentFlags: flags.NewComponentFlags(), outputFlags: flags.NewOutputFlags(), concurrencyFlags: flags.NewConcurrencyFlags(), junitPath: "", + htmlPath: "", } cmd := &cobra.Command{ @@ -88,6 +92,14 @@ func NewRunSuiteCommand(registry *extension.Registry) *cobra.Command { } compositeWriter.AddWriter(junitWriter) } + // HTML writer if needed + if opts.htmlPath != "" { + htmlWriter, err := extensiontests.NewHTMLResultWriter(opts.htmlPath, suite.Name) + if err != nil { + return errors.Wrap(err, "couldn't create html writer") + } + compositeWriter.AddWriter(htmlWriter) + } // JSON writer jsonWriter, err := extensiontests.NewJSONResultWriter(os.Stdout, @@ -102,13 +114,40 @@ func NewRunSuiteCommand(registry *extension.Registry) *cobra.Command { return errors.Wrap(err, "couldn't filter specs") } - return specs.Run(ctx, compositeWriter, opts.concurrencyFlags.MaxConcurency) + concurrency := opts.concurrencyFlags.MaxConcurency + if suite.Parallelism > 0 { + concurrency = min(concurrency, suite.Parallelism) + } + results, runErr := specs.Run(ctx, compositeWriter, concurrency) + if opts.junitPath != "" { + // we want to commit the results to disk regardless of the success or failure of the specs + if err := writeResults(opts.junitPath, results); err != nil { + fmt.Fprintf(os.Stderr, "Failed to write test results to disk: %v\n", err) + } + } + return runErr }, } opts.componentFlags.BindFlags(cmd.Flags()) opts.outputFlags.BindFlags(cmd.Flags()) opts.concurrencyFlags.BindFlags(cmd.Flags()) cmd.Flags().StringVarP(&opts.junitPath, "junit-path", "j", opts.junitPath, "write results to junit XML") + cmd.Flags().StringVar(&opts.htmlPath, "html-path", opts.htmlPath, "write results to summary HTML") return cmd } + +func writeResults(jUnitPath string, results []*extensiontests.ExtensionTestResult) error { + jUnitDir := filepath.Dir(jUnitPath) + if err := os.MkdirAll(jUnitDir, 0755); err != nil { + return fmt.Errorf("failed to create output directory: %v", err) + } + + encodedResults, err := json.MarshalIndent(results, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal results: %v", err) + } + + outputPath := filepath.Join(jUnitDir, fmt.Sprintf("extension_test_result_e2e_%s.json", time.Now().UTC().Format("20060102-150405"))) + return os.WriteFile(outputPath, encodedResults, 0644) +} diff --git a/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdrun/runtest.go b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdrun/runtest.go index c06894ed9972..c62021e7ecec 100644 --- a/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdrun/runtest.go +++ b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdrun/runtest.go @@ -3,9 +3,9 @@ package cmdrun import ( "bufio" "context" + "errors" "fmt" "os" - "errors" "os/signal" "syscall" "time" @@ -100,7 +100,8 @@ func NewRunTestCommand(registry *extension.Registry) *cobra.Command { } defer w.Flush() - return specs.Run(ctx, w, opts.concurrencyFlags.MaxConcurency) + _, err = specs.Run(ctx, w, opts.concurrencyFlags.MaxConcurency) + return err }, } opts.componentFlags.BindFlags(cmd.Flags()) diff --git a/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/result.go b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/result.go index 2e36969fe6bc..9c03a0a84bab 100644 --- a/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/result.go +++ b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/result.go @@ -1,8 +1,12 @@ package extensiontests import ( + "bytes" + _ "embed" + "encoding/json" "fmt" "strings" + "text/template" "github.com/openshift-eng/openshift-tests-extension/pkg/junit" ) @@ -67,3 +71,55 @@ func (results ExtensionTestResults) ToJUnit(suiteName string) junit.TestSuite { return suite } + +//go:embed viewer.html +var viewerHtml []byte + +// RenderResultsHTML renders the HTML viewer template with the provided JSON data. +// The caller is responsible for marshaling their results to JSON. This allows +// callers with different result struct types to use the same HTML viewer. +func RenderResultsHTML(jsonData []byte, suiteName string) ([]byte, error) { + tmpl, err := template.New("viewer").Parse(string(viewerHtml)) + if err != nil { + return nil, fmt.Errorf("failed to parse template: %w", err) + } + var out bytes.Buffer + if err := tmpl.Execute(&out, struct { + Data string + SuiteName string + }{ + string(jsonData), + suiteName, + }); err != nil { + return nil, fmt.Errorf("failed to execute template: %w", err) + } + return out.Bytes(), nil +} + +func (results ExtensionTestResults) ToHTML(suiteName string) ([]byte, error) { + encoded, err := json.Marshal(results) + if err != nil { + return nil, fmt.Errorf("failed to marshal extension test results: %w", err) + } + // pare down the output if there's a lot, we want this to load in some reasonable amount of time + if len(encoded) > 2<<20 { + // n.b. this is wasteful, but we want to mutate our inputs in a safe manner, so the encode/decode/encode + // pass is useful as a deep copy + var copiedResults ExtensionTestResults + if err := json.Unmarshal(encoded, &copiedResults); err != nil { + return nil, fmt.Errorf("failed to unmarshal extension test results: %w", err) + } + copiedResults.Walk(func(result *ExtensionTestResult) { + if result.Result == ResultPassed { + result.Error = "" + result.Output = "" + result.Details = nil + } + }) + encoded, err = json.Marshal(copiedResults) + if err != nil { + return nil, fmt.Errorf("failed to marshal extension test results: %w", err) + } + } + return RenderResultsHTML(encoded, suiteName) +} diff --git a/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/result_writer.go b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/result_writer.go index aedc409c1795..f9ca434cad84 100644 --- a/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/result_writer.go +++ b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/result_writer.go @@ -124,8 +124,9 @@ func NewJSONResultWriter(out io.Writer, format ResultFormat) (*JSONResultWriter, } return &JSONResultWriter{ - out: out, - format: format, + out: out, + format: format, + results: ExtensionTestResults{}, }, nil } @@ -162,3 +163,51 @@ func (w *JSONResultWriter) Flush() error { return nil } + +type HTMLResultWriter struct { + lock sync.Mutex + testSuite *junit.TestSuite + out *os.File + suiteName string + path string + results ExtensionTestResults +} + +func NewHTMLResultWriter(path, suiteName string) (ResultWriter, error) { + file, err := os.Create(path) + if err != nil { + return nil, err + } + + return &HTMLResultWriter{ + testSuite: &junit.TestSuite{ + Name: suiteName, + }, + out: file, + suiteName: suiteName, + path: path, + }, nil +} + +func (w *HTMLResultWriter) Write(res *ExtensionTestResult) { + w.lock.Lock() + defer w.lock.Unlock() + w.results = append(w.results, res) +} + +func (w *HTMLResultWriter) Flush() error { + w.lock.Lock() + defer w.lock.Unlock() + data, err := w.results.ToHTML(w.suiteName) + if err != nil { + return fmt.Errorf("failed to create result HTML: %w", err) + } + if _, err := w.out.Write(data); err != nil { + return err + } + if err := w.out.Close(); err != nil { + return err + } + + return nil +} \ No newline at end of file diff --git a/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/spec.go b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/spec.go index 4e77842451d7..e87809c8a2a1 100644 --- a/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/spec.go +++ b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/spec.go @@ -196,7 +196,7 @@ func (specs ExtensionTestSpecs) Names() []string { // are written to the given ResultWriter after each spec has completed execution. BeforeEach, // BeforeAll, AfterEach, AfterAll hooks are executed when specified. "Each" hooks must be thread // safe. Returns an error if any test spec failed, indicating the quantity of failures. -func (specs ExtensionTestSpecs) Run(ctx context.Context, w ResultWriter, maxConcurrent int) error { +func (specs ExtensionTestSpecs) Run(ctx context.Context, w ResultWriter, maxConcurrent int) ([]*ExtensionTestResult, error) { queue := make(chan *ExtensionTestSpec) terminalFailures := atomic.Int64{} nonTerminalFailures := atomic.Int64{} @@ -224,6 +224,7 @@ func (specs ExtensionTestSpecs) Run(ctx context.Context, w ResultWriter, maxConc // Start consumers var wg sync.WaitGroup + resultChan := make(chan *ExtensionTestResult, len(specs)) for i := 0; i < maxConcurrent; i++ { wg.Add(1) go func() { @@ -250,12 +251,14 @@ func (specs ExtensionTestSpecs) Run(ctx context.Context, w ResultWriter, maxConc // it does, we may want to modify it (e.g. k8s-tests for annotations currently). res.Name = spec.Name w.Write(res) + resultChan <- res } }() } // Wait for all consumers to finish wg.Wait() + close(resultChan) // Execute afterAll for _, spec := range specs { @@ -264,6 +267,11 @@ func (specs ExtensionTestSpecs) Run(ctx context.Context, w ResultWriter, maxConc } } + var results []*ExtensionTestResult + for res := range resultChan { + results = append(results, res) + } + terminalFailCount := terminalFailures.Load() nonTerminalFailCount := nonTerminalFailures.Load() @@ -275,12 +283,12 @@ func (specs ExtensionTestSpecs) Run(ctx context.Context, w ResultWriter, maxConc // Only exit with error if terminal lifecycle tests failed if terminalFailCount > 0 { if nonTerminalFailCount > 0 { - return fmt.Errorf("%d tests failed (%d informing)", terminalFailCount+nonTerminalFailCount, nonTerminalFailCount) + return results, fmt.Errorf("%d tests failed (%d informing)", terminalFailCount+nonTerminalFailCount, nonTerminalFailCount) } - return fmt.Errorf("%d tests failed", terminalFailCount) + return results, fmt.Errorf("%d tests failed", terminalFailCount) } - return nil + return results, nil } // AddBeforeAll adds a function to be run once before all tests start executing. diff --git a/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/viewer.html b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/viewer.html new file mode 100644 index 000000000000..2ff236aa3271 --- /dev/null +++ b/vendor/github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests/viewer.html @@ -0,0 +1,1520 @@ + + + + + + Results for {{ .SuiteName }} + + + +
+
+

+ + + + + Results for {{ .SuiteName }} +

+

No file loaded

+
+ +
+

Load Test Results

+

Drag and drop a JSON test results file here, or click to browse

+ +
+ + + + +
+ + + + + diff --git a/vendor/modules.txt b/vendor/modules.txt index 617f2a880d2a..f8e2618a4cea 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1422,7 +1422,7 @@ github.com/opencontainers/runtime-spec/specs-go github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label github.com/opencontainers/selinux/pkg/pwalkdir -# github.com/openshift-eng/openshift-tests-extension v0.0.0-20251113163031-356b66aa5c24 +# github.com/openshift-eng/openshift-tests-extension v0.0.0-20251218142942-7ecc8801b9df ## explicit; go 1.23.0 github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdinfo github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdlist