Skip to content

Commit 247aa01

Browse files
stbenjamstbenjam
committed
feat(test): add OTE metadata and test source tracking to JUnit results
Extend JUnit test case schema with OpenShift Test Extensions (OTE) metadata attributes. Metadata fields added: - Lifecycle classification (blocking/informing) - Test execution timestamps (start-time, end-time) - Source tracking (source-image, source-binary, source-url, source-commit) The source fields allow tracking which container image, binary, repository, and commit produced each test result, enabling better traceability and debugging of test failures across different builds.
1 parent 5c72de0 commit 247aa01

File tree

3 files changed

+322
-11
lines changed

3 files changed

+322
-11
lines changed

‎pkg/test/ginkgo/junit.go‎

Lines changed: 47 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,38 @@ import (
1010
"time"
1111

1212
"github.com/openshift/origin/pkg/test"
13+
"github.com/openshift/origin/pkg/test/extensions"
1314
"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
1415

1516
"github.com/openshift/origin/pkg/version"
1617
)
1718

19+
// populateOTEMetadata adds OTE metadata attributes to a JUnit test case if available
20+
func populateOTEMetadata(testCase *junitapi.JUnitTestCase, extensionResult *extensions.ExtensionTestResult) {
21+
if extensionResult == nil {
22+
return
23+
}
24+
25+
// Test source information
26+
testCase.SourceImage = extensionResult.Source.SourceImage
27+
testCase.SourceBinary = extensionResult.Source.SourceBinary
28+
testCase.SourceURL = extensionResult.Source.SourceURL
29+
testCase.SourceCommit = extensionResult.Source.Commit
30+
31+
// Set lifecycle attribute
32+
testCase.Lifecycle = string(extensionResult.Lifecycle)
33+
34+
// Set start time attribute if available
35+
if extensionResult.StartTime != nil {
36+
testCase.StartTime = time.Time(*extensionResult.StartTime).UTC().Format(time.RFC3339)
37+
}
38+
39+
// Set end time attribute if available
40+
if extensionResult.EndTime != nil {
41+
testCase.EndTime = time.Time(*extensionResult.EndTime).UTC().Format(time.RFC3339)
42+
}
43+
}
44+
1845
func generateJUnitTestSuiteResults(
1946
name string,
2047
duration time.Duration,
@@ -36,49 +63,59 @@ func generateJUnitTestSuiteResults(
3663
case test.skipped:
3764
s.NumTests++
3865
s.NumSkipped++
39-
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
66+
testCase := &junitapi.JUnitTestCase{
4067
Name: test.name,
4168
SystemOut: string(test.testOutputBytes),
4269
Duration: test.duration.Seconds(),
4370
SkipMessage: &junitapi.SkipMessage{
4471
Message: lastLinesUntil(string(test.testOutputBytes), 100, "skip ["),
4572
},
46-
})
73+
}
74+
populateOTEMetadata(testCase, test.extensionTestResult)
75+
s.TestCases = append(s.TestCases, testCase)
4776
case test.failed:
4877
s.NumTests++
4978
s.NumFailed++
50-
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
79+
testCase := &junitapi.JUnitTestCase{
5180
Name: test.name,
5281
SystemOut: string(test.testOutputBytes),
5382
Duration: test.duration.Seconds(),
5483
FailureOutput: &junitapi.FailureOutput{
5584
Output: lastLinesUntil(string(test.testOutputBytes), 100, "fail ["),
5685
},
57-
})
86+
}
87+
populateOTEMetadata(testCase, test.extensionTestResult)
88+
s.TestCases = append(s.TestCases, testCase)
5889
case test.flake:
5990
s.NumTests++
6091
s.NumFailed++
61-
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
92+
failedTestCase := &junitapi.JUnitTestCase{
6293
Name: test.name,
6394
SystemOut: string(test.testOutputBytes),
6495
Duration: test.duration.Seconds(),
6596
FailureOutput: &junitapi.FailureOutput{
6697
Output: lastLinesUntil(string(test.testOutputBytes), 100, "flake:"),
6798
},
68-
})
99+
}
100+
populateOTEMetadata(failedTestCase, test.extensionTestResult)
101+
s.TestCases = append(s.TestCases, failedTestCase)
69102

70103
// also add the successful junit result:
71104
s.NumTests++
72-
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
105+
successTestCase := &junitapi.JUnitTestCase{
73106
Name: test.name,
74107
Duration: test.duration.Seconds(),
75-
})
108+
}
109+
populateOTEMetadata(successTestCase, test.extensionTestResult)
110+
s.TestCases = append(s.TestCases, successTestCase)
76111
case test.success:
77112
s.NumTests++
78-
s.TestCases = append(s.TestCases, &junitapi.JUnitTestCase{
113+
testCase := &junitapi.JUnitTestCase{
79114
Name: test.name,
80115
Duration: test.duration.Seconds(),
81-
})
116+
}
117+
populateOTEMetadata(testCase, test.extensionTestResult)
118+
s.TestCases = append(s.TestCases, testCase)
82119
}
83120
}
84121
for _, result := range syntheticTestResults {

‎pkg/test/ginkgo/junit_test.go‎

Lines changed: 266 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
package ginkgo
22

3-
import "testing"
3+
import (
4+
"encoding/xml"
5+
"testing"
6+
"time"
7+
8+
"github.com/openshift-eng/openshift-tests-extension/pkg/dbtime"
9+
"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
10+
"github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
11+
"github.com/openshift/origin/pkg/test/extensions"
12+
"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
13+
)
414

515
func Test_lastLines(t *testing.T) {
616
tests := []struct {
@@ -33,3 +43,258 @@ func Test_lastLines(t *testing.T) {
3343
})
3444
}
3545
}
46+
47+
func Test_populateOTEMetadata(t *testing.T) {
48+
startTime := time.Date(2023, 12, 25, 10, 0, 0, 0, time.UTC)
49+
endTime := time.Date(2023, 12, 25, 10, 5, 0, 0, time.UTC)
50+
51+
tests := []struct {
52+
name string
53+
extensionResult *extensions.ExtensionTestResult
54+
expectedLifecycle string
55+
expectedStartTime string
56+
expectedEndTime string
57+
expectedSourceImage string
58+
expectedSourceBinary string
59+
expectedSourceURL string
60+
expectedSourceCommit string
61+
}{
62+
{
63+
name: "nil extension result",
64+
extensionResult: nil,
65+
expectedLifecycle: "",
66+
expectedStartTime: "",
67+
expectedEndTime: "",
68+
expectedSourceImage: "",
69+
expectedSourceBinary: "",
70+
expectedSourceURL: "",
71+
expectedSourceCommit: "",
72+
},
73+
{
74+
name: "complete extension result",
75+
extensionResult: &extensions.ExtensionTestResult{
76+
ExtensionTestResult: &extensiontests.ExtensionTestResult{
77+
Name: "test-case",
78+
Lifecycle: extensiontests.LifecycleBlocking,
79+
StartTime: dbtime.Ptr(startTime),
80+
EndTime: dbtime.Ptr(endTime),
81+
},
82+
Source: extensions.Source{
83+
Source: &extension.Source{
84+
Commit: "abc123def456",
85+
SourceURL: "https://github.com/example/repo",
86+
},
87+
SourceImage: "tests",
88+
SourceBinary: "openshift-tests",
89+
},
90+
},
91+
expectedLifecycle: "blocking",
92+
expectedStartTime: "2023-12-25T10:00:00Z",
93+
expectedEndTime: "2023-12-25T10:05:00Z",
94+
expectedSourceImage: "tests",
95+
expectedSourceBinary: "openshift-tests",
96+
expectedSourceURL: "https://github.com/example/repo",
97+
expectedSourceCommit: "abc123def456",
98+
},
99+
{
100+
name: "informing lifecycle",
101+
extensionResult: &extensions.ExtensionTestResult{
102+
ExtensionTestResult: &extensiontests.ExtensionTestResult{
103+
Name: "test-case",
104+
Lifecycle: extensiontests.LifecycleInforming,
105+
StartTime: dbtime.Ptr(startTime),
106+
EndTime: dbtime.Ptr(endTime),
107+
},
108+
Source: extensions.Source{
109+
Source: &extension.Source{
110+
Commit: "xyz789",
111+
SourceURL: "https://github.com/openshift/origin",
112+
},
113+
SourceImage: "tests",
114+
SourceBinary: "openshift-tests",
115+
},
116+
},
117+
expectedLifecycle: "informing",
118+
expectedStartTime: "2023-12-25T10:00:00Z",
119+
expectedEndTime: "2023-12-25T10:05:00Z",
120+
expectedSourceImage: "tests",
121+
expectedSourceBinary: "openshift-tests",
122+
expectedSourceURL: "https://github.com/openshift/origin",
123+
expectedSourceCommit: "xyz789",
124+
},
125+
{
126+
name: "missing time fields",
127+
extensionResult: &extensions.ExtensionTestResult{
128+
ExtensionTestResult: &extensiontests.ExtensionTestResult{
129+
Name: "test-case",
130+
Lifecycle: extensiontests.LifecycleBlocking,
131+
StartTime: nil,
132+
EndTime: nil,
133+
},
134+
},
135+
expectedLifecycle: "blocking",
136+
expectedStartTime: "",
137+
expectedEndTime: "",
138+
expectedSourceImage: "",
139+
expectedSourceBinary: "",
140+
expectedSourceURL: "",
141+
expectedSourceCommit: "",
142+
},
143+
{
144+
name: "partial source information",
145+
extensionResult: &extensions.ExtensionTestResult{
146+
ExtensionTestResult: &extensiontests.ExtensionTestResult{
147+
Name: "test-case",
148+
Lifecycle: extensiontests.LifecycleBlocking,
149+
StartTime: dbtime.Ptr(startTime),
150+
EndTime: dbtime.Ptr(endTime),
151+
},
152+
Source: extensions.Source{
153+
Source: &extension.Source{
154+
Commit: "abc123",
155+
},
156+
SourceImage: "tests",
157+
},
158+
},
159+
expectedLifecycle: "blocking",
160+
expectedStartTime: "2023-12-25T10:00:00Z",
161+
expectedEndTime: "2023-12-25T10:05:00Z",
162+
expectedSourceImage: "tests",
163+
expectedSourceBinary: "",
164+
expectedSourceURL: "",
165+
expectedSourceCommit: "abc123",
166+
},
167+
}
168+
169+
for _, tt := range tests {
170+
t.Run(tt.name, func(t *testing.T) {
171+
testCase := &junitapi.JUnitTestCase{
172+
Name: "test-case",
173+
Duration: 300.0, // 5 minutes
174+
}
175+
176+
populateOTEMetadata(testCase, tt.extensionResult)
177+
178+
if testCase.Lifecycle != tt.expectedLifecycle {
179+
t.Errorf("Lifecycle = %q, want %q", testCase.Lifecycle, tt.expectedLifecycle)
180+
}
181+
if testCase.StartTime != tt.expectedStartTime {
182+
t.Errorf("StartTime = %q, want %q", testCase.StartTime, tt.expectedStartTime)
183+
}
184+
if testCase.EndTime != tt.expectedEndTime {
185+
t.Errorf("EndTime = %q, want %q", testCase.EndTime, tt.expectedEndTime)
186+
}
187+
if testCase.SourceImage != tt.expectedSourceImage {
188+
t.Errorf("SourceImage = %q, want %q", testCase.SourceImage, tt.expectedSourceImage)
189+
}
190+
if testCase.SourceBinary != tt.expectedSourceBinary {
191+
t.Errorf("SourceBinary = %q, want %q", testCase.SourceBinary, tt.expectedSourceBinary)
192+
}
193+
if testCase.SourceURL != tt.expectedSourceURL {
194+
t.Errorf("SourceURL = %q, want %q", testCase.SourceURL, tt.expectedSourceURL)
195+
}
196+
if testCase.SourceCommit != tt.expectedSourceCommit {
197+
t.Errorf("SourceCommit = %q, want %q", testCase.SourceCommit, tt.expectedSourceCommit)
198+
}
199+
})
200+
}
201+
}
202+
203+
func Test_junitXMLWithOTEAttributes(t *testing.T) {
204+
startTime := time.Date(2023, 12, 25, 10, 0, 0, 0, time.UTC)
205+
endTime := time.Date(2023, 12, 25, 10, 5, 0, 0, time.UTC)
206+
207+
// Create a JUnit test case and populate it with OTE metadata
208+
extensionResult := &extensions.ExtensionTestResult{
209+
ExtensionTestResult: &extensiontests.ExtensionTestResult{
210+
Name: "example-test",
211+
Lifecycle: extensiontests.LifecycleBlocking,
212+
StartTime: dbtime.Ptr(startTime),
213+
EndTime: dbtime.Ptr(endTime),
214+
},
215+
Source: extensions.Source{
216+
Source: &extension.Source{
217+
Commit: "abc123def456789",
218+
SourceURL: "https://github.com/openshift/origin",
219+
},
220+
SourceImage: "tests",
221+
SourceBinary: "openshift-tests",
222+
},
223+
}
224+
225+
junitTestCase := &junitapi.JUnitTestCase{
226+
Name: "example-test",
227+
Duration: 300.0, // 5 minutes
228+
}
229+
230+
// Populate the OTE metadata
231+
populateOTEMetadata(junitTestCase, extensionResult)
232+
233+
// Create a test suite containing our test case
234+
suite := &junitapi.JUnitTestSuite{
235+
Name: "test-suite",
236+
NumTests: 1,
237+
Duration: 300.0,
238+
TestCases: []*junitapi.JUnitTestCase{junitTestCase},
239+
}
240+
241+
// Verify OTE metadata is present
242+
if junitTestCase.Lifecycle != "blocking" {
243+
t.Errorf("Lifecycle = %q, want %q", junitTestCase.Lifecycle, "blocking")
244+
}
245+
if junitTestCase.StartTime != "2023-12-25T10:00:00Z" {
246+
t.Errorf("StartTime = %q, want %q", junitTestCase.StartTime, "2023-12-25T10:00:00Z")
247+
}
248+
if junitTestCase.EndTime != "2023-12-25T10:05:00Z" {
249+
t.Errorf("EndTime = %q, want %q", junitTestCase.EndTime, "2023-12-25T10:05:00Z")
250+
}
251+
if junitTestCase.SourceImage != "tests" {
252+
t.Errorf("SourceImage = %q, want %q", junitTestCase.SourceImage, "tests")
253+
}
254+
if junitTestCase.SourceBinary != "openshift-tests" {
255+
t.Errorf("SourceBinary = %q, want %q", junitTestCase.SourceBinary, "openshift-tests")
256+
}
257+
if junitTestCase.SourceURL != "https://github.com/openshift/origin" {
258+
t.Errorf("SourceURL = %q, want %q", junitTestCase.SourceURL, "https://github.com/openshift/origin")
259+
}
260+
if junitTestCase.SourceCommit != "abc123def456789" {
261+
t.Errorf("SourceCommit = %q, want %q", junitTestCase.SourceCommit, "abc123def456789")
262+
}
263+
264+
// Verify XML marshaling includes the new attributes
265+
xmlData, err := xml.MarshalIndent(suite, "", " ")
266+
if err != nil {
267+
t.Fatalf("Failed to marshal XML: %v", err)
268+
}
269+
270+
xmlString := string(xmlData)
271+
272+
// Check that our custom attributes are in the XML
273+
expectedAttributes := []string{
274+
`lifecycle="blocking"`,
275+
`start-time="2023-12-25T10:00:00Z"`,
276+
`end-time="2023-12-25T10:05:00Z"`,
277+
`source-image="tests"`,
278+
`source-binary="openshift-tests"`,
279+
`source-url="https://github.com/openshift/origin"`,
280+
`source-commit="abc123def456789"`,
281+
}
282+
283+
for _, attr := range expectedAttributes {
284+
if !contains(xmlString, attr) {
285+
t.Errorf("XML does not contain expected attribute: %s\nXML output:\n%s", attr, xmlString)
286+
}
287+
}
288+
}
289+
290+
// Helper function to check if a string contains a substring
291+
func contains(s, substr string) bool {
292+
return len(s) >= len(substr) && func() bool {
293+
for i := 0; i <= len(s)-len(substr); i++ {
294+
if s[i:i+len(substr)] == substr {
295+
return true
296+
}
297+
}
298+
return false
299+
}()
300+
}

0 commit comments

Comments
 (0)