Skip to content

Commit c2a089a

Browse files
smuppandsmuppand
committed
Probe_Failure_Check: tighten probe failure detection and CI hints
Improve Probe_Failure_Check/run.sh to catch more realistic probe/firmware issues in CI/CD: - Extend FAIL_PATTERN to match: - "deferred probe timeout, ignoring dependency" - "probe with driver <drv> failed ..." while keeping all existing probe/firmware/bind patterns. - Capture a kernel log snapshot and emit up to 10 representative CI-HINT lines to stdout/stderr for easier debugging. - If available, read /sys/kernel/debug/devices_deferred and: - Log entries into probe_failures.log - Emit DEFERRED: lines for unresolved deferred probes. Signed-off-by: Srikanth Muppandam <smuppand@qti.qualcomm.com>
1 parent c305809 commit c2a089a

File tree

1 file changed

+123
-34
lines changed
  • Runner/suites/Kernel/Baseport/Probe_Failure_Check

1 file changed

+123
-34
lines changed

‎Runner/suites/Kernel/Baseport/Probe_Failure_Check/run.sh‎

Lines changed: 123 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,148 @@
11
#!/bin/sh
2-
32
# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
43
# SPDX-License-Identifier: BSD-3-Clause-Clear
4+
# Probe failure / deferred probe detector using kernel logs + devices_deferred
55

6-
# Robustly source init_env and functestlib.sh
7-
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
6+
# ---------- Repo env + helpers ----------
7+
SCRIPT_DIR="$(
8+
cd "$(dirname "$0")" || exit 1
9+
pwd
10+
)"
811
INIT_ENV=""
912
SEARCH="$SCRIPT_DIR"
13+
1014
while [ "$SEARCH" != "/" ]; do
11-
if [ -f "$SEARCH/init_env" ]; then
12-
INIT_ENV="$SEARCH/init_env"
13-
break
14-
fi
15-
SEARCH=$(dirname "$SEARCH")
15+
if [ -f "$SEARCH/init_env" ]; then
16+
INIT_ENV="$SEARCH/init_env"
17+
break
18+
fi
19+
SEARCH=$(dirname "$SEARCH")
1620
done
1721

1822
if [ -z "$INIT_ENV" ]; then
19-
echo "[ERROR] Could not find init_env (starting at $SCRIPT_DIR)" >&2
20-
exit 1
23+
echo "[ERROR] Could not find init_env (starting at $SCRIPT_DIR)" >&2
24+
exit 1
2125
fi
2226

23-
if [ -z "$__INIT_ENV_LOADED" ]; then
24-
# shellcheck disable=SC1090
25-
. "$INIT_ENV"
27+
# Only source once (idempotent)
28+
# NOTE: We intentionally **do not export** any new vars. They stay local to this shell.
29+
if [ -z "${__INIT_ENV_LOADED:-}" ]; then
30+
# shellcheck disable=SC1090
31+
. "$INIT_ENV"
32+
__INIT_ENV_LOADED=1
2633
fi
2734

35+
# Keep combined suppression for consistency across repo
2836
# shellcheck disable=SC1090,SC1091
2937
. "$TOOLS/functestlib.sh"
3038

3139
TESTNAME="Probe_Failure_Check"
40+
RESULT_FILE="$TESTNAME.res"
41+
LOG_FILE="probe_failures.log"
42+
43+
# Move into testcase directory (so .res and logs land in the right place)
3244
test_path=$(find_test_case_by_name "$TESTNAME")
3345
cd "$test_path" || exit 1
3446

35-
res_file="./$TESTNAME.res"
36-
log_file="./probe_failures.log"
37-
38-
log_info "-----------------------------------------------------------------------------------------"
47+
log_info "----------------------------------------------------------------------------"
3948
log_info "------------------- Starting $TESTNAME Testcase ----------------------------"
4049

41-
rm -f "$res_file" "$log_file"
42-
{
43-
echo "Probe Failure Report - $(date)"
44-
echo "--------------------------------------------------"
45-
} > "$log_file"
46-
47-
if get_kernel_log 2>/dev/null | \
48-
grep -iE '([[:alnum:]_.-]+:)?[[:space:]]*(probe failed|failed to probe|probe error)' \
49-
>> "$log_file"; then
50-
log_error "Probe failures detected; see $log_file"
51-
log_fail "$TESTNAME : Probe failures found"
52-
echo "$TESTNAME FAIL" > "$res_file"
53-
exit 1
50+
rm -f "$RESULT_FILE" "$LOG_FILE"
51+
: >"$LOG_FILE"
52+
53+
# --- Get kernel log snapshot ---
54+
if command -v get_kernel_log >/dev/null 2>&1; then
55+
KERNEL_LOG="$(get_kernel_log 2>/dev/null)"
56+
else
57+
log_warn "'get_kernel_log' not found, falling back to 'dmesg -T'"
58+
KERNEL_LOG="$(dmesg -T 2>/dev/null)"
59+
fi
60+
61+
# --- SKIP when unable to collect kernel logs ---
62+
if [ -z "$KERNEL_LOG" ]; then
63+
if command -v log_skip >/dev/null 2>&1; then
64+
log_skip "$TESTNAME : Unable to collect kernel logs, skipping probe failure check"
65+
else
66+
log_warn "$TESTNAME : Unable to collect kernel logs, treating as SKIP"
67+
fi
68+
echo "$TESTNAME SKIP" >"$RESULT_FILE"
69+
rm -f "$LOG_FILE"
70+
exit 0
71+
fi
72+
73+
# --- Probe / firmware / bind failure patterns ---
74+
# Intentionally broad but targeted at realistic driver init / teardown problems.
75+
# Built as a concatenation of single-quoted segments to avoid stray backslashes.
76+
FAIL_PATTERN='(failed to (probe|instantiate)|'\
77+
'probe of .* failed|probe with driver .* failed|deferred probe timeout, ignoring dependency|'\
78+
'Direct firmware load .* failed|tplg firmware loading .* failed|ASoC error|'\
79+
'unprobe failed|failed to remove .* driver|component bind error|component unbind error|'\
80+
'Driver .* failed|cannot register device driver|register_component error|'\
81+
'sound: ASoC: failed|load_firmware failed|request_firmware failed|device_add failed|'\
82+
'platform device_add failed|Cannot register component|bind failed|'\
83+
'lookup_component error|failed to add machine driver)'
84+
85+
# Pull out matched lines
86+
MATCHES=$(printf '%s\n' "$KERNEL_LOG" | grep -Ei "$FAIL_PATTERN" || true)
87+
88+
# --- Optional: report devices still in deferred-probe list (debugfs) ---
89+
DEFERRED_FILE="/sys/kernel/debug/devices_deferred"
90+
if [ -r "$DEFERRED_FILE" ]; then
91+
DEFERRED_CONTENT=$(cat "$DEFERRED_FILE" 2>/dev/null || true)
92+
if [ -n "$DEFERRED_CONTENT" ]; then
93+
log_warn "Devices still listed in $DEFERRED_FILE (deferred probe not resolved):"
94+
printf '%s\n' "$DEFERRED_CONTENT" >>"$LOG_FILE"
95+
printf '%s\n' "$DEFERRED_CONTENT" | while IFS= read -r line; do
96+
[ -n "$line" ] || continue
97+
log_warn "DEFERRED: $line"
98+
done
99+
else
100+
log_info "No entries in $DEFERRED_FILE (no outstanding deferred probes)."
101+
fi
54102
else
55-
rm -f "$log_file"
56-
log_pass "$TESTNAME : No probe failures found"
57-
echo "$TESTNAME PASS" > "$res_file"
58-
exit 0
103+
log_info "$DEFERRED_FILE not available (no deferred-probe debugfs support)."
104+
fi
105+
106+
# --- Evaluate matches and report to CI ---
107+
if [ -n "$MATCHES" ]; then
108+
# Save full match set to log file for post-mortem
109+
printf '%s\n' "$MATCHES" >>"$LOG_FILE"
110+
111+
MATCH_COUNT=$(printf '%s\n' "$MATCHES" | wc -l | awk '{print $1}')
112+
# Extract the latest leading [timestamp] if present, e.g. "[ 10.471969]"
113+
LATEST_TS=$(
114+
printf '%s\n' "$MATCHES" \
115+
| sed -n 's/^\(\[[^]]*]\).*/\1/p' \
116+
| tail -n 1
117+
)
118+
119+
log_fail "$TESTNAME : Kernel probe/unprobe/firmware-related errors found (see $LOG_FILE)"
120+
log_info "Total matched lines: $MATCH_COUNT"
121+
if [ -n "$LATEST_TS" ]; then
122+
log_info "Latest timestamp among matches: $LATEST_TS"
123+
else
124+
log_info "Latest timestamp among matches: Not Available"
125+
fi
126+
127+
# Print a few representative lines to stdout for CI log visibility
128+
printf '%s\n' "$MATCHES" | head -n 10 | while IFS= read -r line; do
129+
[ -n "$line" ] || continue
130+
echo "CI-HINT: $line"
131+
done
132+
133+
# Dump entire LOG_FILE for easy inspection in LAVA logs
134+
if [ -s "$LOG_FILE" ]; then
135+
echo "================ $LOG_FILE (full contents) ================"
136+
cat "$LOG_FILE"
137+
echo "================ end of $LOG_FILE ========================="
138+
fi
139+
140+
echo "$TESTNAME FAIL" >"$RESULT_FILE"
141+
# Convention: exit 0, result is driven by .res file
142+
exit 0
59143
fi
144+
145+
log_pass "$TESTNAME : No probe/firmware/bind errors found in kernel log snapshot"
146+
echo "$TESTNAME PASS" >"$RESULT_FILE"
147+
rm -f "$LOG_FILE"
148+
exit 0

0 commit comments

Comments
 (0)