Skip to content

Commit 19d38c1

Browse files
peter-marcisovskypeter-marcisovsky
committed
feat(ci): Build idf examples with all overridden components
 - all component used in usb device and host examples are overridden - esp_tinyusb, usb and all class drivers - using ignore_build_warnings.txt from esp-idf to match the idf CI environment
1 parent f84e76a commit 19d38c1

File tree

4 files changed

+197
-40
lines changed

4 files changed

+197
-40
lines changed

‎.github/ci/.idf-build-examples-rules.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,6 @@ examples/peripherals/usb/device/tusb_ncm:
1515
- if: SOC_USB_OTG_SUPPORTED != 1 or SOC_WIFI_SUPPORTED != 1
1616

1717
examples/peripherals/usb/host:
18-
enable:
19-
- if: (IDF_VERSION >= "5.4.0")
20-
reason: Run USB Host examples with overridden (esp-usb) USB component only for service releases
2118
disable:
2219
- if: SOC_USB_OTG_SUPPORTED != 1
2320

‎.github/ci/.idf_build_examples_config.toml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,11 @@ target = "all"
1111

1212
# build related options
1313
build_dir = "build_@t_@w"
14-
work_dir = "@f_@t_@w"
14+
15+
# use ignore_build_warnings.txt from esp-idf
16+
ignore_warning_files = [
17+
'tools/ci/ignore_build_warnings.txt',
18+
]
1519

1620
# config rules
1721
config_rules = [

‎.github/ci/override_managed_component.py‎

Lines changed: 163 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -3,81 +3,213 @@
33
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
44
# SPDX-License-Identifier: Apache-2.0
55

6+
"""
7+
Managed components overriding
8+
------------------------------------
9+
10+
The script:
11+
- finds all examples on specified path, based on presence of idf_component.yml inside main folder
12+
- skips CherryUSB example
13+
- creates idf_component.yml for usb_host_lib example, as it does not have any
14+
- override all the components defined in the example's idf_component.yml
15+
with a local paths to the components present in esp-usb
16+
- it can separately override esp_tinyusb, usb (usb_host_lib) or all class drivers
17+
18+
Usage Examples:
19+
---------------
20+
21+
To override esp_tinyusb component in usb device examples:
22+
override_managed_component.py esp_tinyusb device/esp_tinyusb ${IDF_PATH}/examples/peripherals/usb/device/*
23+
24+
To override usb (usb_host_lib) component in usb host examples:
25+
override_managed_component.py usb host/usb ${IDF_PATH}/examples/peripherals/usb/host/*
26+
27+
To override all class drivers in usb host examples:
28+
override_managed_component.py class host/class ${IDF_PATH}/examples/peripherals/usb/host/*
29+
"""
30+
631
import sys
732
import argparse
833
import yaml
934
from pathlib import Path
1035
from glob import glob
1136
from idf_component_tools.manager import ManifestManager
1237

13-
def find_apps_with_manifest(apps):
38+
39+
def _has_manifest(path: Path) -> bool:
40+
"""Check if a given directory contains main/idf_component.yml"""
41+
return (path / "main" / "idf_component.yml").exists()
42+
43+
def _create_usb_host_lib_manifest(path: Path) -> None:
44+
"""Ensure usb_host_lib has a manifest file"""
45+
main_dir = path / "main"
46+
main_dir.mkdir(parents=True, exist_ok=True)
47+
manifest_file = main_dir / "idf_component.yml"
48+
if not manifest_file.exists():
49+
try:
50+
print(f"[Info] Creating idf_component.yml for usb_host_lib: {manifest_file}")
51+
with open(manifest_file, "w", encoding="utf8") as f:
52+
f.write("dependencies: {}\n")
53+
except (OSError, PermissionError) as e:
54+
print(f"[Error] Failed to create manifest {manifest_file}: {e}")
55+
raise
56+
57+
def find_apps_with_manifest(component, apps):
1458
"""
1559
Given a list of paths or glob patterns, return a list of directories
1660
that contain 'main/idf_component.yml'. Checks one level of subfolders.
61+
Excludes any path that contains 'CherryUSB'.
1762
"""
1863
apps_with_glob = []
1964

2065
for app in apps:
2166
# Expand wildcards
2267
for match in glob(app):
68+
if 'cherryusb' in match:
69+
continue # Skip CHerryUSB examples, path containing 'cherryusb'
70+
2371
p = Path(match)
72+
if not p.is_dir():
73+
continue
2474

25-
if p.is_dir():
26-
# Check main/idf_component.yml directly inside
27-
manifest = p / "main" / "idf_component.yml"
28-
if manifest.exists():
29-
apps_with_glob.append(str(p))
30-
continue # already found, no need to check subfolders
75+
# Check main/idf_component.yml directly inside
76+
if _has_manifest(p):
77+
apps_with_glob.append(str(p))
78+
continue # already found, no need to check subfolders
3179

32-
# Check one level of subfolders
33-
for sub in p.iterdir():
34-
if sub.is_dir():
35-
manifest = sub / "main" / "idf_component.yml"
36-
if manifest.exists():
37-
apps_with_glob.append(str(sub))
80+
# Check one level of subfolders
81+
for sub in p.iterdir():
82+
if sub.is_dir() and _has_manifest(sub):
83+
apps_with_glob.append(str(sub))
84+
85+
# Special handling: create empty manifest for usb_host_lib if missing
86+
if component == "usb":
87+
for app in apps:
88+
for match in glob(app):
89+
p = Path(match)
90+
if p.is_dir() and p.name == "usb_host_lib":
91+
_create_usb_host_lib_manifest(p)
92+
apps_with_glob.append(str(p))
3893

3994
return apps_with_glob
4095

96+
def _get_class_name(app_path: Path) -> str:
97+
"""Determine class name from example path"""
98+
known_classes = {"cdc", "hid", "msc", "uvc"}
99+
parts = app_path.parts
100+
101+
if parts and parts[-1] in known_classes:
102+
return app_path.parts[-1]
103+
elif len(parts) >= 2 and parts[-2] in known_classes:
104+
return app_path.parts[-2]
105+
else:
106+
raise ValueError(f"Could not determine class name from path: {app_path}")
107+
108+
41109
def override_with_local_component(component, local_path, app):
110+
"""Override managed component with local component"""
42111
app_path = Path(app)
43-
44112
absolute_local_path = Path(local_path).absolute()
113+
45114
if not absolute_local_path.exists():
46-
print('[Error] {} path does not exist'.format(local_path))
47-
raise Exception
115+
raise FileNotFoundError(f'[Error] {local_path} path does not exist')
48116
if not app_path.exists():
49-
print('[Error] {} path does not exist'.format(app_path))
50-
raise Exception
117+
raise FileNotFoundError(f'[Error] {app_path} path does not exist')
51118

52-
print('[Info] Processing app {}'.format(app))
119+
print(f'[Info] Processing app {app}')
53120
manager = ManifestManager(app_path / 'main', 'app')
54-
if '/' not in component:
55-
# Prepend with default namespace
56-
component_with_namespace = 'espressif/' + component
121+
122+
# Normalize main component name
123+
component_with_namespace = component if '/' in component else f'espressif/{component}'
57124

58125
try:
59-
manifest_tree = yaml.safe_load(Path(manager.path).read_text())
60-
manifest_tree['dependencies'][component_with_namespace] = {
126+
manifest_tree = yaml.safe_load(Path(manager.path).read_text(encoding='utf8'))
127+
except yaml.YAMLError as e:
128+
raise FileNotFoundError(f"[Error] Failed to parse manifest {manager.path}: {e}") from e
129+
deps = manifest_tree.get('dependencies', {})
130+
131+
# --- Override esp_tinyusb component in usb device examples ---
132+
if component == "esp_tinyusb":
133+
deps[component_with_namespace] = {
134+
'version': '*',
135+
'override_path': str(absolute_local_path)
136+
}
137+
print(f"[Info] Overridden esp_tinyusb {component_with_namespace} -> {absolute_local_path}")
138+
139+
# --- Override usb component in usb host examples ---
140+
if component == "usb":
141+
deps[component_with_namespace] = {
61142
'version': '*',
62143
'override_path': str(absolute_local_path)
63144
}
64-
with open(manager.path, 'w') as f:
145+
print(f"[Info] Overridden usb {component_with_namespace} -> {absolute_local_path}")
146+
147+
# --- Override all class drivers components in usb host examples ---
148+
if component == "class":
149+
# Determine class name from example path
150+
class_name = _get_class_name(app_path)
151+
152+
for usb_host_dep in [dep for dep in deps if dep.split("/")[-1].startswith("usb_host_")]:
153+
short_name = usb_host_dep.split("/")[-1]
154+
local_subpath = absolute_local_path.parent / "class" / class_name / short_name
155+
if not local_subpath.exists():
156+
raise FileNotFoundError(f"[Error] Local path for {usb_host_dep} not found at {local_subpath}")
157+
158+
usb_host_dep_with_ns = usb_host_dep if '/' in usb_host_dep else f'espressif/{usb_host_dep}'
159+
160+
# Remove old key if it’s the non-namespaced one
161+
if usb_host_dep_with_ns != usb_host_dep and usb_host_dep in deps:
162+
deps.pop(usb_host_dep)
163+
164+
deps[usb_host_dep_with_ns] = {
165+
'version': '*',
166+
'override_path': str(local_subpath)
167+
}
168+
print(f"[Info] Overridden class {usb_host_dep} -> {local_subpath}")
169+
170+
# Special override for cdc_acm_vcp examples
171+
if app_path.name == "cdc_acm_vcp":
172+
extra_dep = "usb_host_cdc_acm"
173+
extra_path = absolute_local_path.parent / "class" / class_name / extra_dep
174+
if not extra_path.exists():
175+
raise FileNotFoundError(f"[Error] Local path for {extra_dep} not found at {extra_path}")
176+
177+
extra_dep_with_ns = f'espressif/{extra_dep}'
178+
deps[extra_dep_with_ns] = {
179+
'version': '*',
180+
'override_path': str(extra_path)
181+
}
182+
print(f"[Info] Overridden extra component {extra_dep} -> {extra_path}")
183+
184+
manifest_tree['dependencies'] = deps
185+
186+
try:
187+
with open(manager.path, 'w', encoding='utf8') as f:
65188
yaml.dump(manifest_tree, f, allow_unicode=True, Dumper=yaml.SafeDumper)
66-
except KeyError:
67-
print('[Error] {} app does not depend on {}'.format(app, component_with_namespace))
68-
raise KeyError
189+
except (OSError, PermissionError) as e:
190+
print(f"[Error] Failed to write manifest {manager.path}: {e}")
191+
raise
192+
69193

70194
def override_with_local_component_all(component, local_path, apps):
195+
"""Find apps and override components"""
196+
71197
# Process wildcard, e.g. "app_prefix_*"
72-
apps_with_glob = find_apps_with_manifest(apps)
198+
apps_with_glob = find_apps_with_manifest(component, apps)
199+
200+
print("[Info] Apps found:")
201+
for app in apps_with_glob:
202+
print(f"[Info] {app}")
73203

74204
# Go through all collected apps
75205
for app in apps_with_glob:
76206
try:
77207
override_with_local_component(component, local_path, app)
78-
except:
79-
print("[Error] Could not process app {}".format(app))
208+
except Exception as e:
209+
print(f"[Error] Could not process app {app}: {e}")
80210
return -1
211+
212+
print("[Info] Overriding complete")
81213
return 0
82214

83215

‎.github/workflows/build_idf_examples.yml‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,11 @@
33
# - usb device examples: with overridden esp_tinyusb from esp-usb/device/esp_tinyusb
44
# - Only IDF Latest
55
#
6-
# - usb host examples: with overridden usb component from esp-usb/host/usb
7-
# - Only service IDF releases
6+
# - usb host examples:
7+
# - Overridden usb component from esp-usb/host/usb and overridden class drivers from esp-usb/host/class
8+
# - Only service IDF releases
9+
# - Overridden class drivers from esp-usb/host/class
10+
# - All (service + maintenance IDF releases)
811

912
name: Build ESP-IDF USB examples
1013

@@ -31,6 +34,7 @@ jobs:
3134
env:
3235
CONFIG_PATH: ${{ github.workspace }}/.github/ci/.idf_build_examples_config.toml
3336
MANIFEST_PATH: ${{ github.workspace }}/.github/ci/.idf-build-examples-rules.yml
37+
EXAMPLES_PATH: ${{ github.workspace }}
3438
steps:
3539
- uses: actions/checkout@v4
3640
with:
@@ -40,14 +44,34 @@ jobs:
4044
run: |
4145
. ${IDF_PATH}/export.sh
4246
pip install idf-component-manager>=2.1.2 idf-build-apps==2.12.2 pyyaml --upgrade
47+
- name: Setup IDF Examples path
48+
run: echo "EXAMPLES_PATH=${IDF_PATH}/examples/peripherals/usb" >> $GITHUB_ENV
49+
- name: Override device component
50+
if: contains('latest', matrix.idf_ver)
51+
run: |
52+
. ${IDF_PATH}/export.sh
53+
# Device examples have been updated to use esp_tinyusb 2.x only on esp-idf latest for now, TODO IDF-14282
54+
#python .github/ci/override_managed_component.py esp_tinyusb device/esp_tinyusb ${{ env.EXAMPLES_PATH }}/device/*
55+
- name: Override class components
56+
run: |
57+
. ${IDF_PATH}/export.sh
58+
python .github/ci/override_managed_component.py class host/class ${{ env.EXAMPLES_PATH }}/host/*
59+
- name: Override usb component
60+
if: contains('release-v5.4 release-v5.5 latest', matrix.idf_ver)
61+
run: |
62+
# Override usb component only for maintenance releases
63+
. ${IDF_PATH}/export.sh
64+
python .github/ci/override_managed_component.py usb host/usb ${{ env.EXAMPLES_PATH }}/host/*
4365
- name: Build ESP-IDF ${{ matrix.idf_ver }} USB examples
4466
shell: bash
4567
run: |
4668
. ${IDF_PATH}/export.sh
47-
48-
python .github/ci/override_managed_component.py esp_tinyusb device/esp_tinyusb ${IDF_PATH}/examples/peripherals/usb/device/*
49-
python .github/ci/override_managed_component.py usb host/usb ${IDF_PATH}/examples/peripherals/usb/host/*
5069
cd ${IDF_PATH}
5170
71+
# Export compiler flags
72+
export PEDANTIC_FLAGS="-DIDF_CI_BUILD -Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function"
73+
export EXTRA_CFLAGS="${PEDANTIC_FLAGS} -Wstrict-prototypes"
74+
export EXTRA_CXXFLAGS="${PEDANTIC_FLAGS}"
75+
5276
idf-build-apps find --config-file ${CONFIG_PATH} --manifest-file ${MANIFEST_PATH}
5377
idf-build-apps build --config-file ${CONFIG_PATH} --manifest-file ${MANIFEST_PATH}

0 commit comments

Comments
 (0)