Skip to content

Commit 5d84f2c

Browse files
lucasssvazlucasssvaz
committed
change(ci): Overhaul CI test flow management
1 parent b20655a commit 5d84f2c

File tree

507 files changed

+2129
-1937
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

507 files changed

+2129
-1937
lines changed

‎.github/CODEOWNERS‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,9 @@
7272
/libraries/Wire/ @me-no-dev
7373
/libraries/Zigbee/ @P-R-O-C-H-Y
7474

75-
# CI JSON
75+
# CI YAML
7676
# Keep this after other libraries and tests to avoid being overridden.
77-
**/ci.json @lucasssvaz
77+
**/ci.yml @lucasssvaz
7878

7979
# The CODEOWNERS file should be owned by the developers of the ESP32 Arduino Core.
8080
# Leave this entry as the last one to avoid being overridden.

‎.github/scripts/generate_missing_junits.py‎

Lines changed: 322 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,322 @@
1+
#!/usr/bin/env python3
2+
3+
import json
4+
import os
5+
import re
6+
import sys
7+
from pathlib import Path
8+
from xml.etree.ElementTree import Element, SubElement, ElementTree
9+
import yaml
10+
11+
12+
def parse_array(value) -> list[str]:
13+
if isinstance(value, list):
14+
return [str(x) for x in value]
15+
if not isinstance(value, str):
16+
return []
17+
txt = value.strip()
18+
if not txt:
19+
return []
20+
# Try JSON
21+
try:
22+
return [str(x) for x in json.loads(txt)]
23+
except Exception:
24+
pass
25+
# Normalize single quotes then JSON
26+
try:
27+
fixed = txt.replace("'", '"')
28+
return [str(x) for x in json.loads(fixed)]
29+
except Exception:
30+
pass
31+
# Fallback: CSV
32+
return [p.strip() for p in txt.strip("[]").split(",") if p.strip()]
33+
34+
35+
def _parse_ci_yml(content: str) -> dict:
36+
if not content:
37+
return {}
38+
try:
39+
data = yaml.safe_load(content) or {}
40+
if not isinstance(data, dict):
41+
return {}
42+
return data
43+
except Exception:
44+
return {}
45+
46+
47+
def _fqbn_counts_from_yaml(ci: dict) -> dict[str, int]:
48+
counts: dict[str, int] = {}
49+
if not isinstance(ci, dict):
50+
return counts
51+
fqbn = ci.get("fqbn")
52+
if not isinstance(fqbn, dict):
53+
return counts
54+
for target, entries in fqbn.items():
55+
if isinstance(entries, list):
56+
counts[str(target)] = len(entries)
57+
elif entries is not None:
58+
# Single value provided as string
59+
counts[str(target)] = 1
60+
return counts
61+
62+
63+
def _sdkconfig_meets(ci_cfg: dict, sdk_text: str) -> bool:
64+
if not sdk_text:
65+
return True
66+
for req in ci_cfg.get("requires", []):
67+
if not req or not isinstance(req, str):
68+
continue
69+
if not any(line.startswith(req) for line in sdk_text.splitlines()):
70+
return False
71+
req_any = ci_cfg.get("requires_any", [])
72+
if req_any:
73+
if not any(any(line.startswith(r.strip()) for line in sdk_text.splitlines()) for r in req_any if isinstance(r, str)):
74+
return False
75+
return True
76+
77+
78+
def expected_from_artifacts(build_root: Path) -> dict[tuple[str, str, str, str], int]:
79+
"""Compute expected runs using ci.yml and sdkconfig found in build artifacts.
80+
Returns mapping (platform, target, type, sketch) -> expected_count
81+
"""
82+
expected: dict[tuple[str, str, str, str], int] = {}
83+
if not build_root.exists():
84+
return expected
85+
print(f"[DEBUG] Scanning build artifacts in: {build_root}", file=sys.stderr)
86+
for artifact_dir in build_root.iterdir():
87+
if not artifact_dir.is_dir():
88+
continue
89+
m = re.match(r"test-bin-([A-Za-z0-9_\-]+)-([A-Za-z0-9_\-]+)", artifact_dir.name)
90+
if not m:
91+
continue
92+
target = m.group(1)
93+
test_type = m.group(2)
94+
print(f"[DEBUG] Artifact group target={target} type={test_type} dir={artifact_dir}", file=sys.stderr)
95+
96+
# Group build*.tmp directories by sketch
97+
# Structure: test-bin-<target>-<type>/<sketch>/build*.tmp/
98+
sketches_processed = set()
99+
100+
# Find all build*.tmp directories and process each sketch once
101+
for build_tmp in artifact_dir.rglob("build*.tmp"):
102+
if not build_tmp.is_dir():
103+
continue
104+
if not re.search(r"build\d*\.tmp$", build_tmp.name):
105+
continue
106+
107+
# Path structure is: test-bin-<target>-<type>/<sketch>/build*.tmp/
108+
sketch = build_tmp.parent.name
109+
110+
# Skip if we already processed this sketch
111+
if sketch in sketches_processed:
112+
continue
113+
sketches_processed.add(sketch)
114+
115+
print(f"[DEBUG] Processing sketch={sketch} from artifact {artifact_dir.name}", file=sys.stderr)
116+
117+
ci_path = build_tmp / "ci.yml"
118+
sdk_path = build_tmp / "sdkconfig"
119+
120+
# Read ci.yml if it exists, otherwise use empty (defaults)
121+
ci_text = ""
122+
if ci_path.exists():
123+
try:
124+
ci_text = ci_path.read_text(encoding="utf-8")
125+
except Exception as e:
126+
print(f"[DEBUG] Warning: failed to read ci.yml: {e}", file=sys.stderr)
127+
else:
128+
print(f"[DEBUG] No ci.yml found, using defaults", file=sys.stderr)
129+
130+
try:
131+
sdk_text = sdk_path.read_text(encoding="utf-8", errors="ignore") if sdk_path.exists() else ""
132+
except Exception:
133+
sdk_text = ""
134+
135+
ci = _parse_ci_yml(ci_text)
136+
fqbn_counts = _fqbn_counts_from_yaml(ci)
137+
138+
# Determine allowed platforms for this test
139+
# Performance tests are only run on hardware
140+
if test_type == "performance":
141+
allowed_platforms = ["hardware"]
142+
else:
143+
allowed_platforms = []
144+
platforms_cfg = ci.get("platforms") if isinstance(ci, dict) else None
145+
for plat in ("hardware", "wokwi", "qemu"):
146+
dis = None
147+
if isinstance(platforms_cfg, dict):
148+
dis = platforms_cfg.get(plat)
149+
if dis is False:
150+
continue
151+
allowed_platforms.append(plat)
152+
153+
# Requirements check
154+
minimal = {
155+
"requires": ci.get("requires") or [],
156+
"requires_any": ci.get("requires_any") or [],
157+
}
158+
if not _sdkconfig_meets(minimal, sdk_text):
159+
print(f"[DEBUG] Skip (requirements not met): target={target} type={test_type} sketch={sketch}", file=sys.stderr)
160+
continue
161+
162+
# Expected runs = number from fqbn_counts in ci.yml (how many FQBNs for this target)
163+
exp_runs = fqbn_counts.get(target, 0) or 1
164+
print(f"[DEBUG] ci.yml specifies {exp_runs} FQBN(s) for target={target}", file=sys.stderr)
165+
166+
for plat in allowed_platforms:
167+
expected[(plat, target, test_type, sketch)] = exp_runs
168+
print(f"[DEBUG] Expected: plat={plat} target={target} type={test_type} sketch={sketch} runs={exp_runs}", file=sys.stderr)
169+
170+
if len(sketches_processed) == 0:
171+
print(f"[DEBUG] No sketches found in this artifact group", file=sys.stderr)
172+
return expected
173+
174+
175+
def scan_executed_xml(xml_root: Path, valid_types: set[str]) -> dict[tuple[str, str, str, str], int]:
176+
"""Return executed counts per (platform, target, type, sketch).
177+
Type/sketch/target are inferred from .../<type>/<sketch>/<target>/<file>.xml
178+
"""
179+
counts: dict[tuple[str, str, str, str], int] = {}
180+
if not xml_root.exists():
181+
print(f"[DEBUG] Results root not found: {xml_root}", file=sys.stderr)
182+
return counts
183+
print(f"[DEBUG] Scanning executed XMLs in: {xml_root}", file=sys.stderr)
184+
for xml_path in xml_root.rglob("*.xml"):
185+
if not xml_path.is_file():
186+
continue
187+
rel = str(xml_path)
188+
platform = "hardware"
189+
if "test-results-wokwi-" in rel:
190+
platform = "wokwi"
191+
elif "test-results-qemu-" in rel:
192+
platform = "qemu"
193+
# Expect .../<type>/<sketch>/<target>/*.xml
194+
parts = xml_path.parts
195+
t_idx = -1
196+
for i, p in enumerate(parts):
197+
if p in valid_types:
198+
t_idx = i
199+
if t_idx == -1 or t_idx + 3 >= len(parts):
200+
continue
201+
test_type = parts[t_idx]
202+
sketch = parts[t_idx + 1]
203+
target = parts[t_idx + 2]
204+
key = (platform, target, test_type, sketch)
205+
old_count = counts.get(key, 0)
206+
counts[key] = old_count + 1
207+
print(f"[DEBUG] Executed XML #{old_count + 1}: plat={platform} target={target} type={test_type} sketch={sketch} file={xml_path.name}", file=sys.stderr)
208+
print(f"[DEBUG] Executed entries discovered: {len(counts)}", file=sys.stderr)
209+
return counts
210+
211+
212+
def write_missing_xml(out_root: Path, platform: str, target: str, test_type: str, sketch: str, missing_count: int):
213+
out_tests_dir = out_root / f"test-results-{platform}" / "tests" / test_type / sketch / target
214+
out_tests_dir.mkdir(parents=True, exist_ok=True)
215+
# Create one XML per missing index
216+
for idx in range(missing_count):
217+
suite_name = f"{test_type}_{platform}_{target}_{sketch}"
218+
root = Element("testsuite", name=suite_name, tests="1", failures="0", errors="1")
219+
case = SubElement(root, "testcase", classname=f"{test_type}.{sketch}", name="missing-run")
220+
error = SubElement(case, "error", message="Expected test run missing")
221+
error.text = "This placeholder indicates an expected test run did not execute."
222+
tree = ElementTree(root)
223+
out_file = out_tests_dir / f"{sketch}_missing_{idx}.xml"
224+
tree.write(out_file, encoding="utf-8", xml_declaration=True)
225+
226+
227+
def main():
228+
# Args: <build_artifacts_dir> <test_results_dir> <output_junit_dir>
229+
if len(sys.argv) != 4:
230+
print(f"Usage: {sys.argv[0]} <build_artifacts_dir> <test_results_dir> <output_junit_dir>", file=sys.stderr)
231+
return 2
232+
233+
build_root = Path(sys.argv[1]).resolve()
234+
results_root = Path(sys.argv[2]).resolve()
235+
out_root = Path(sys.argv[3]).resolve()
236+
237+
# Validate inputs
238+
if not build_root.is_dir():
239+
print(f"ERROR: Build artifacts directory not found: {build_root}", file=sys.stderr)
240+
return 2
241+
if not results_root.is_dir():
242+
print(f"ERROR: Test results directory not found: {results_root}", file=sys.stderr)
243+
return 2
244+
# Ensure output directory exists
245+
try:
246+
out_root.mkdir(parents=True, exist_ok=True)
247+
except Exception as e:
248+
print(f"ERROR: Failed to create output directory {out_root}: {e}", file=sys.stderr)
249+
return 2
250+
251+
# Read matrices from environment variables injected by workflow
252+
hw_enabled = (os.environ.get("HW_TESTS_ENABLED", "false").lower() == "true")
253+
wokwi_enabled = (os.environ.get("WOKWI_TESTS_ENABLED", "false").lower() == "true")
254+
qemu_enabled = (os.environ.get("QEMU_TESTS_ENABLED", "false").lower() == "true")
255+
256+
hw_targets = parse_array(os.environ.get("HW_TARGETS", "[]"))
257+
wokwi_targets = parse_array(os.environ.get("WOKWI_TARGETS", "[]"))
258+
qemu_targets = parse_array(os.environ.get("QEMU_TARGETS", "[]"))
259+
260+
hw_types = parse_array(os.environ.get("HW_TYPES", "[]"))
261+
wokwi_types = parse_array(os.environ.get("WOKWI_TYPES", "[]"))
262+
qemu_types = parse_array(os.environ.get("QEMU_TYPES", "[]"))
263+
264+
expected = expected_from_artifacts(build_root) # (platform, target, type, sketch) -> expected_count
265+
executed_types = set(hw_types + wokwi_types + qemu_types)
266+
executed = scan_executed_xml(results_root, executed_types) # (platform, target, type, sketch) -> count
267+
print(f"[DEBUG] Expected entries computed: {len(expected)}", file=sys.stderr)
268+
269+
# Filter expected by enabled platforms and target/type matrices
270+
enabled_plats = set()
271+
if hw_enabled:
272+
enabled_plats.add("hardware")
273+
if wokwi_enabled:
274+
enabled_plats.add("wokwi")
275+
if qemu_enabled:
276+
enabled_plats.add("qemu")
277+
278+
# Build platform-specific target and type sets
279+
plat_targets = {
280+
"hardware": set(hw_targets),
281+
"wokwi": set(wokwi_targets),
282+
"qemu": set(qemu_targets),
283+
}
284+
plat_types = {
285+
"hardware": set(hw_types),
286+
"wokwi": set(wokwi_types),
287+
"qemu": set(qemu_types),
288+
}
289+
290+
missing_total = 0
291+
extra_total = 0
292+
for (plat, target, test_type, sketch), exp_count in expected.items():
293+
if plat not in enabled_plats:
294+
continue
295+
# Check if target and type are valid for this specific platform
296+
if target not in plat_targets.get(plat, set()):
297+
continue
298+
if test_type not in plat_types.get(plat, set()):
299+
continue
300+
got = executed.get((plat, target, test_type, sketch), 0)
301+
if got < exp_count:
302+
print(f"[DEBUG] Missing: plat={plat} target={target} type={test_type} sketch={sketch} expected={exp_count} got={got}", file=sys.stderr)
303+
write_missing_xml(out_root, plat, target, test_type, sketch, exp_count - got)
304+
missing_total += (exp_count - got)
305+
elif got > exp_count:
306+
print(f"[DEBUG] Extra runs: plat={plat} target={target} type={test_type} sketch={sketch} expected={exp_count} got={got}", file=sys.stderr)
307+
extra_total += (got - exp_count)
308+
309+
# Check for executed tests that were not expected at all
310+
for (plat, target, test_type, sketch), got in executed.items():
311+
if (plat, target, test_type, sketch) not in expected:
312+
print(f"[DEBUG] Unexpected test: plat={plat} target={target} type={test_type} sketch={sketch} got={got} (not in expected)", file=sys.stderr)
313+
314+
print(f"Generated {missing_total} placeholder JUnit files for missing runs.", file=sys.stderr)
315+
if extra_total > 0:
316+
print(f"WARNING: {extra_total} extra test runs detected (more than expected).", file=sys.stderr)
317+
318+
319+
if __name__ == "__main__":
320+
sys.exit(main())
321+
322+

‎.github/scripts/get_affected.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@
6363
Build file patterns
6464
--------------------
6565
- **build_files**: Core Arduino build system files (platform.txt, variants/**, etc.)
66-
- **sketch_build_files**: Sketch-specific files (ci.json, *.csv in example directories)
66+
- **sketch_build_files**: Sketch-specific files (ci.yml, *.csv in example directories)
6767
- **idf_build_files**: Core IDF build system files (CMakeLists.txt, idf_component.yml, etc.)
6868
- **idf_project_files**: Project-specific IDF files (per-example CMakeLists.txt, sdkconfig, etc.)
6969
@@ -128,7 +128,7 @@
128128
# Files that are used by the sketch build system.
129129
# If any of these files change, the sketch should be recompiled.
130130
sketch_build_files = [
131-
"libraries/*/examples/**/ci.json",
131+
"libraries/*/examples/**/ci.yml",
132132
"libraries/*/examples/**/*.csv",
133133
]
134134

@@ -150,7 +150,7 @@
150150
# If any of these files change, the example that uses them should be recompiled.
151151
idf_project_files = [
152152
"idf_component_examples/*/CMakeLists.txt",
153-
"idf_component_examples/*/ci.json",
153+
"idf_component_examples/*/ci.yml",
154154
"idf_component_examples/*/*.csv",
155155
"idf_component_examples/*/sdkconfig*",
156156
"idf_component_examples/*/main/*",

‎.github/scripts/on-push-idf.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ fi
1717

1818
for example in $affected_examples; do
1919
example_path="$PWD/components/arduino-esp32/$example"
20-
if [ -f "$example_path/ci.json" ]; then
20+
if [ -f "$example_path/ci.yml" ]; then
2121
# If the target is listed as false, skip the sketch. Otherwise, include it.
22-
is_target=$(jq -r --arg target "$IDF_TARGET" '.targets[$target]' "$example_path/ci.json")
22+
is_target=$(yq eval ".targets.${IDF_TARGET}" "$example_path/ci.yml" 2>/dev/null)
2323
if [[ "$is_target" == "false" ]]; then
2424
printf "\n\033[93mSkipping %s for target %s\033[0m\n\n" "$example" "$IDF_TARGET"
2525
continue

0 commit comments

Comments
 (0)