From f32a2d1d791ef5af17590ee2f70c08adb8dc8ec1 Mon Sep 17 00:00:00 2001 From: Christian Schilling Date: Tue, 27 Jan 2026 12:51:58 +0100 Subject: [PATCH] Add cli based integration test This is the beginning of porting the integration tests from pytest to cram/prysk/scrut for way easier test authoring and maintanance. Change-Id: I25ac890818d91616362c6a56328a27d4615d3e3b --- .github/workflows/run-pytest-on-posix.yml | 12 ++- docker/development/Dockerfile | 3 + docker/development/files/requirements.lock | 28 +++--- docker/development/files/requirements.txt | 7 ++ test/setup.sh | 29 ++++++ tests/cli/can_command.t | 15 +++ tests/cli/can_echo.t | 18 ++++ tests/cli/ethernet.t | 13 +++ tests/cli/lc_reboot.t | 17 ++++ tests/cli/logger.t | 102 +++++++++++++++++++++ tests/cli/run.sh | 2 + tests/cli/storage.t | 47 ++++++++++ tests/cli/uds_tester_present.t | 10 ++ 13 files changed, 286 insertions(+), 17 deletions(-) create mode 100644 test/setup.sh create mode 100644 tests/cli/can_command.t create mode 100644 tests/cli/can_echo.t create mode 100644 tests/cli/ethernet.t create mode 100644 tests/cli/lc_reboot.t create mode 100644 tests/cli/logger.t create mode 100644 tests/cli/run.sh create mode 100644 tests/cli/storage.t create mode 100644 tests/cli/uds_tester_present.t diff --git a/.github/workflows/run-pytest-on-posix.yml b/.github/workflows/run-pytest-on-posix.yml index a77bbcba456..b88414e666a 100644 --- a/.github/workflows/run-pytest-on-posix.yml +++ b/.github/workflows/run-pytest-on-posix.yml @@ -46,7 +46,17 @@ jobs: run: | sudo tools/enet/bring-up-ethernet.sh - - name: Run the pytest inside the docker container + - name: Run the pytest integration tests inside the docker container run: >- DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) DOCKER_HISTORY=/dev/null docker compose run --rm development-sil python3 .ci/posix_pytest.py + + - name: Run the cram integration tests for freertos inside the docker container + run: >- + DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) DOCKER_HISTORY=/dev/null docker compose run --rm development-sil + bash -lc 'cd tests/cli && REFAPP="$PWD/../../build/posix-freertos/executables/referenceApp/application/Release/app.referenceApp.elf" bash run.sh' + + - name: Run the cram integration tests for threadx inside the docker container + run: >- + DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) DOCKER_HISTORY=/dev/null docker compose run --rm development-sil + bash -lc 'cd tests/cli && REFAPP="$PWD/../../build/posix-threadx/executables/referenceApp/application/Release/app.referenceApp.elf" bash run.sh' diff --git a/docker/development/Dockerfile b/docker/development/Dockerfile index 54d8cd2a2ab..9eef07b2038 100644 --- a/docker/development/Dockerfile +++ b/docker/development/Dockerfile @@ -42,6 +42,8 @@ RUN apt-get update && apt-get install -y \ g++-11 \ gcc-11 \ git \ + iproute2 \ + iputils-ping \ lcov \ ninja-build \ python-is-python3 \ @@ -54,6 +56,7 @@ RUN apt-get update && apt-get install -y \ wget \ iproute2\ iputils-ping\ + can-utils \ zip COPY --from=external-downloads /gcc-arm.tar.xz /gcc-arm.tar.xz diff --git a/docker/development/files/requirements.lock b/docker/development/files/requirements.lock index 102164142d1..d151e190ac8 100644 --- a/docker/development/files/requirements.lock +++ b/docker/development/files/requirements.lock @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.14 # by the following command: # -# pip-compile --output-file=docker/development/files/requirements.lock docker/development/files/requirements.txt test/pyTest/requirements.txt tools/UdsTool/requirements.txt +# pip-compile --output-file=docker/development/files/requirements.lock docker/development/files/requirements.txt test/pyTest/requirements.txt # anyio==4.12.0 # via httpx @@ -12,31 +12,31 @@ cachecontrol[filecache]==0.14.4 # via poetry can-isotp==2.0.6 # via + # -r docker/development/files/requirements.txt # -r test/pyTest/requirements.txt - # -r tools/UdsTool/requirements.txt certifi==2025.11.12 # via # httpcore # httpx # requests cffi==2.0.0 - # via cryptography + # via xattr charset-normalizer==3.4.4 # via requests cleo==2.1.0 # via poetry click==8.0.3 - # via -r tools/UdsTool/requirements.txt + # via -r docker/development/files/requirements.txt cmakelang==0.6.13 # via -r docker/development/files/requirements.txt +cram==0.7 + # via -r docker/development/files/requirements.txt crashtest==0.4.1 # via cleo -cryptography==46.0.3 - # via secretstorage distlib==0.4.0 # via virtualenv doipclient==1.1.1 - # via -r tools/UdsTool/requirements.txt + # via -r docker/development/files/requirements.txt dulwich==0.24.10 # via poetry exceptiongroup==1.2.2 @@ -72,10 +72,6 @@ jaraco-context==6.0.1 # via keyring jaraco-functools==4.3.0 # via keyring -jeepney==0.9.0 - # via - # keyring - # secretstorage keyring==25.7.0 # via poetry markdown-it-py==4.0.0 @@ -129,6 +125,7 @@ pytest==8.3.3 # via -r test/pyTest/requirements.txt python-can==4.4.2 # via + # -r docker/development/files/requirements.txt # -r test/pyTest/requirements.txt # -r tools/UdsTool/requirements.txt pyyaml==6.0.3 @@ -144,8 +141,6 @@ requests-toolbelt==1.0.0 # via poetry rich==14.2.0 # via -r docker/development/files/requirements.txt -secretstorage==3.5.0 - # via keyring shellingham==1.5.4 # via poetry six==1.17.0 @@ -161,12 +156,11 @@ trove-classifiers==2025.12.1.14 typing-extensions==4.12.2 # via # -r test/pyTest/requirements.txt - # anyio # python-can udsoncan==1.23.1 # via + # -r docker/development/files/requirements.txt # -r test/pyTest/requirements.txt - # -r tools/UdsTool/requirements.txt urllib3==2.6.2 # via # dulwich @@ -177,6 +171,8 @@ wrapt==1.16.0 # via # -r test/pyTest/requirements.txt # python-can +xattr==1.3.0 + # via poetry zstandard==0.25.0 # via pbs-installer diff --git a/docker/development/files/requirements.txt b/docker/development/files/requirements.txt index e1b0cdae099..4bfaf0378ae 100644 --- a/docker/development/files/requirements.txt +++ b/docker/development/files/requirements.txt @@ -3,3 +3,10 @@ poetry~=2.2 rich~=14.2 sniffio~=1.3 PyYAML~=6.0 +cram~=0.7 +can-isotp==2.0.6 +click==8.0.3 +doipclient==1.1.1 +python-can==4.4.2 +setuptools>=70.0.0 +udsoncan==1.23.1 diff --git a/test/setup.sh b/test/setup.sh new file mode 100644 index 00000000000..8991d64f100 --- /dev/null +++ b/test/setup.sh @@ -0,0 +1,29 @@ +export TESTTMP=${PWD} +tmux kill-server > /dev/null 2>&1 + +declare -A OUT_POSITIONS +OUT_POSITIONS['serial']='1' +OUT_POSITIONS['can']='1' + +wait_output() { timeout 10s tail -n +${OUT_POSITIONS[$1]} -f "$TESTTMP/$1.out" | sed -u "/$2/q" | grep "$2"; } +send_serial() { tmux send-keys -t refApp "$1" Enter; } +advance_output() { OUT_POSITIONS[$1]=$(cat ${TESTTMP}/$1.out | wc -l); } + +if [ -z "${REFAPP:-}" ]; then + echo "REFAPP must be set to the reference application executable path" >&2 + return 1 +fi + +if [ ! -x "$REFAPP" ]; then + echo "Reference app not found or not executable: $REFAPP" >&2 + return 1 +fi + +UDSTOOL=$(find $TESTDIR/../.. -name udsTool.py) +candump vcan0 > "${TESTTMP}/can.out" 2>&1 & +REFAPP_CMD="while true; do ${REFAPP}; done | sed -u 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' | sed -u 's/^[^:]*://' > ${TESTTMP}/serial.out" +tmux new-session -d -s refApp bash -c "$REFAPP_CMD" + +wait_output serial 'DEBUG: Run level 9 done' + +export TARGET_IP="192.168.0.201" diff --git a/tests/cli/can_command.t b/tests/cli/can_command.t new file mode 100644 index 00000000000..e920d787396 --- /dev/null +++ b/tests/cli/can_command.t @@ -0,0 +1,15 @@ +Setup the test enviroment + $ . ${TESTDIR}/../../test/setup.sh + RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc) + +Send a can frame via console command + $ send_serial "can send 0x100 100 2 3 46 7 20" + +Check that we see the frame on the bus + $ wait_output can 'vcan0 100' + vcan0 100 [6] 64 02 03 2E 07 14 + +Same for hex bytes + $ send_serial "can send 0x101 0xA1 0xB2 0xC3 0xD4 0xE5 0xF6 0xA7 0xB8" + $ wait_output can 'vcan0 101' + vcan0 101 [8] A1 B2 C3 D4 E5 F6 A7 B8 diff --git a/tests/cli/can_echo.t b/tests/cli/can_echo.t new file mode 100644 index 00000000000..765b5d7766b --- /dev/null +++ b/tests/cli/can_echo.t @@ -0,0 +1,18 @@ +Setup the test enviroment + $ . ${TESTDIR}/../../test/setup.sh + RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc) + +Send a CAN frame to the echo service in the referenceApp + $ cansend vcan0 "123#AABBCCDDEEFF" + +Check that we see the message being sent on the bus + $ wait_output can 'vcan0 123' + vcan0 123 [6] AA BB CC DD EE FF + +Check that we see notification about the reception of the message on the serial out + $ wait_output serial 'received CAN frame, id=0x123, length=6' + RefApp: CAN: DEBUG: [SocketCanTransceiver] received CAN frame, id=0x123, length=6\r (esc) + +Check that we see the echo response on the bus + $ wait_output can 'vcan0 124' + vcan0 124 [6] AA BB CC DD EE FF diff --git a/tests/cli/ethernet.t b/tests/cli/ethernet.t new file mode 100644 index 00000000000..bdfd73cb618 --- /dev/null +++ b/tests/cli/ethernet.t @@ -0,0 +1,13 @@ +Setup the test enviroment + $ . ${TESTDIR}/../../test/setup.sh + RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc) + +Just send two pings and see that we do get a response + $ ping -c 2 -W 2 $TARGET_IP + PING *.*.*.* (*.*.*.*) 56(84) bytes of data. (glob) + 64 bytes from *.*.*.*: icmp_seq=1 ttl=255 time=* ms (glob) + 64 bytes from *.*.*.*: icmp_seq=2 ttl=255 time=* ms (glob) + + --- *.*.*.* ping statistics --- (glob) + 2 packets transmitted, 2 received, 0% packet loss, time * (glob) + rtt min/avg/max/mdev = */*/*/* ms (glob) diff --git a/tests/cli/lc_reboot.t b/tests/cli/lc_reboot.t new file mode 100644 index 00000000000..2f4cb7c5bcb --- /dev/null +++ b/tests/cli/lc_reboot.t @@ -0,0 +1,17 @@ +Setup the test enviroment + $ . ${TESTDIR}/../../test/setup.sh + RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc) + +Advance so we don't match previous output + $ advance_output serial + +Send reboot command + $ send_serial "lc reboot" + +Check that we have reached shutdown + $ wait_output serial 'Lifecycle shutdown complete' + RefApp: LIFECYCLE: INFO: Lifecycle shutdown complete\r (esc) + +And also check that we booted again + $ wait_output serial 'Run level 8 done' + RefApp: LIFECYCLE: DEBUG: Run level 8 done\r (esc) diff --git a/tests/cli/logger.t b/tests/cli/logger.t new file mode 100644 index 00000000000..993f525186f --- /dev/null +++ b/tests/cli/logger.t @@ -0,0 +1,102 @@ +Setup the test enviroment + $ . ${TESTDIR}/../../test/setup.sh + RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc) + + $ advance_output serial + $ send_serial "logger level" + +Use the "logger level" command to check that it reports all components initially at "DEBUG" level + $ wait_output serial 'Received console command "logger level"' + RefApp: CONSOLE: INFO: Received console command "logger level"\r (esc) + $ wait_output serial 'BSP DEBUG' + BSP DEBUG\r (esc) + $ wait_output serial 'COMMON DEBUG' + COMMON DEBUG\r (esc) + $ wait_output serial 'DEMO DEBUG' + DEMO DEBUG\r (esc) + $ wait_output serial 'GLOBAL DEBUG' + GLOBAL DEBUG\r (esc) + $ wait_output serial 'LIFECYCLE DEBUG' + LIFECYCLE DEBUG\r (esc) + $ wait_output serial 'CONSOLE DEBUG' + CONSOLE DEBUG\r (esc) + $ wait_output serial 'CAN DEBUG' + CAN DEBUG\r (esc) + $ wait_output serial 'DOCAN DEBUG' + DOCAN DEBUG\r (esc) + $ wait_output serial 'UDS DEBUG' + UDS DEBUG\r (esc) + $ wait_output serial 'TPROUTER DEBUG' + TPROUTER DEBUG\r (esc) + $ wait_output serial 'ok' + ok\r (esc) + $ wait_output serial 'Console command succeeded' + RefApp: CONSOLE: INFO: Console command succeeded\r (esc) + +Set some components to different log levels + $ advance_output serial + $ send_serial "logger level BSP error" + $ wait_output serial 'Console command succeeded' + RefApp: CONSOLE: INFO: Console command succeeded\r (esc) + + $ advance_output serial + $ send_serial "logger level COMMON info" + $ wait_output serial 'Console command succeeded' + RefApp: CONSOLE: INFO: Console command succeeded\r (esc) + + $ advance_output serial + $ send_serial "logger level DEMO critical" + $ wait_output serial 'Console command succeeded' + RefApp: CONSOLE: INFO: Console command succeeded\r (esc) + +Check the level changes are reported + $ advance_output serial + $ send_serial "logger level" + $ wait_output serial 'BSP ERROR' + BSP ERROR\r (esc) + $ wait_output serial 'COMMON INFO' + COMMON INFO\r (esc) + $ wait_output serial 'DEMO CRITICAL' + DEMO CRITICAL\r (esc) + $ wait_output serial 'GLOBAL DEBUG' + GLOBAL DEBUG\r (esc) + $ wait_output serial 'LIFECYCLE DEBUG' + LIFECYCLE DEBUG\r (esc) + $ wait_output serial 'CONSOLE DEBUG' + CONSOLE DEBUG\r (esc) + $ wait_output serial 'CAN DEBUG' + CAN DEBUG\r (esc) + $ wait_output serial 'DOCAN DEBUG' + DOCAN DEBUG\r (esc) + $ wait_output serial 'UDS DEBUG' + UDS DEBUG\r (esc) + $ wait_output serial 'TPROUTER DEBUG' + TPROUTER DEBUG\r (esc) + $ wait_output serial 'Console command succeeded' + RefApp: CONSOLE: INFO: Console command succeeded\r (esc) + +The CAN logger should only log received frames when in DEBUG. +Verify this by sending a frame both while in DEBUG and while in another +level. + $ advance_output serial + $ advance_output can + $ cansend vcan0 "123#AABBCCDDEEFF" + $ wait_output can 'vcan0 123' + vcan0 123 [6] AA BB CC DD EE FF + $ wait_output serial 'received CAN frame, id=0x123, length=6' + RefApp: CAN: DEBUG: [SocketCanTransceiver] received CAN frame, id=0x123, length=6\r (esc) + $ advance_output serial + $ advance_output can + + $ send_serial "logger level CAN info" + $ wait_output serial 'Console command succeeded' + RefApp: CONSOLE: INFO: Console command succeeded\r (esc) + $ advance_output serial + $ advance_output can + $ cansend vcan0 "123#AABBCCDDEEFF" + $ wait_output can 'vcan0 123' + vcan0 123 [6] AA BB CC DD EE FF + $ wait_output serial 'received CAN frame, id=0x123, length=6' + [1] + $ advance_output serial + $ advance_output can diff --git a/tests/cli/run.sh b/tests/cli/run.sh new file mode 100644 index 00000000000..b6296acb900 --- /dev/null +++ b/tests/cli/run.sh @@ -0,0 +1,2 @@ + +cram --shell=/bin/bash -ivy *.t diff --git a/tests/cli/storage.t b/tests/cli/storage.t new file mode 100644 index 00000000000..4c1fb0eb443 --- /dev/null +++ b/tests/cli/storage.t @@ -0,0 +1,47 @@ +Setup the test enviroment + $ . ${TESTDIR}/../../test/setup.sh + RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc) + +This tests non-volatile storage read/write via console command. +We write a value, reboot the target and read it back to check it survived +a lifecycle reboot. +This is repeated with two different data values at the same address to make +sure we are not passing this test with a leftover written value from a previous +test run. + + $ advance_output serial + $ send_serial "storage write 2561 aabbccdd" + $ wait_output serial 'Console command succeeded' + RefApp: CONSOLE: INFO: Console command succeeded\r (esc) + $ send_serial "storage read 2561 4" + $ wait_output serial "aabbccdd " + aabbccdd \r (no-eol) (esc) + \r (esc) + + $ send_serial "lc reboot" + $ advance_output serial + $ wait_output serial 'Run level 8 done' + RefApp: LIFECYCLE: DEBUG: Run level 8 done\r (esc) + $ send_serial "storage read 2561 4" + $ wait_output serial "aabbccdd " + aabbccdd \r (no-eol) (esc) + \r (esc) + + + $ advance_output serial + $ send_serial "storage write 2561 ddccbbaa" + $ wait_output serial 'Console command succeeded' + RefApp: CONSOLE: INFO: Console command succeeded\r (esc) + $ send_serial "storage read 2561 4" + $ wait_output serial "ddccbbaa " + ddccbbaa \r (no-eol) (esc) + \r (esc) + + $ send_serial "lc reboot" + $ advance_output serial + $ wait_output serial 'Run level 8 done' + RefApp: LIFECYCLE: DEBUG: Run level 8 done\r (esc) + $ send_serial "storage read 2561 4" + $ wait_output serial "ddccbbaa " + ddccbbaa \r (no-eol) (esc) + \r (esc) diff --git a/tests/cli/uds_tester_present.t b/tests/cli/uds_tester_present.t new file mode 100644 index 00000000000..a240ac2b8ba --- /dev/null +++ b/tests/cli/uds_tester_present.t @@ -0,0 +1,10 @@ +Setup the test enviroment + $ . ${TESTDIR}/../../test/setup.sh + RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc) + + $ cansend vcan0 "02A#023E00000000000000" + + $ wait_output can 'vcan0 02A' + vcan0 02A [8] 02 3E 00 00 00 00 00 00 + $ wait_output can 'vcan0 0F0' + vcan0 0F0 [8] 02 7E 00 CC CC CC CC CC