Add cli based integration test

.github/workflows/run-pytest-on-posix.yml

@@ -46,7 +46,17 @@ jobs:
         run: |
           sudo tools/enet/bring-up-ethernet.sh
 
-      - name: Run the pytest inside the docker container
+      - name: Run the pytest integration tests inside the docker container
         run: >-
           DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) DOCKER_HISTORY=/dev/null docker compose run --rm development-sil
           python3 .ci/posix_pytest.py
+
+      - name: Run the cram integration tests for freertos inside the docker container
+        run: >-
+          DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) DOCKER_HISTORY=/dev/null docker compose run --rm development-sil
+          bash -lc 'cd tests/cli && REFAPP="$PWD/../../build/posix-freertos/executables/referenceApp/application/Release/app.referenceApp.elf" bash run.sh'
+
+      - name: Run the cram integration tests for threadx inside the docker container
+        run: >-
+          DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) DOCKER_HISTORY=/dev/null docker compose run --rm development-sil
+          bash -lc 'cd tests/cli && REFAPP="$PWD/../../build/posix-threadx/executables/referenceApp/application/Release/app.referenceApp.elf" bash run.sh'

docker/development/Dockerfile

@@ -42,6 +42,8 @@ RUN apt-get update && apt-get install -y \
         g++-11 \
         gcc-11 \
         git \
+        iproute2 \
+        iputils-ping \
         lcov \
         ninja-build \
         python-is-python3 \
@@ -54,6 +56,7 @@ RUN apt-get update && apt-get install -y \
         wget \
         iproute2\
         iputils-ping\
+        can-utils \
         zip
 
 COPY --from=external-downloads /gcc-arm.tar.xz /gcc-arm.tar.xz

docker/development/files/requirements.lock

@@ -1,8 +1,8 @@
 #
-# This file is autogenerated by pip-compile with Python 3.12
+# This file is autogenerated by pip-compile with Python 3.14
 # by the following command:
 #
-#    pip-compile --output-file=docker/development/files/requirements.lock docker/development/files/requirements.txt test/pyTest/requirements.txt tools/UdsTool/requirements.txt
+#    pip-compile --output-file=docker/development/files/requirements.lock docker/development/files/requirements.txt test/pyTest/requirements.txt
 #
 anyio==4.12.0
     # via httpx
@@ -12,31 +12,31 @@ cachecontrol[filecache]==0.14.4
     # via poetry
 can-isotp==2.0.6
     # via
+    #   -r docker/development/files/requirements.txt
     #   -r test/pyTest/requirements.txt
-    #   -r tools/UdsTool/requirements.txt
 certifi==2025.11.12
     # via
     #   httpcore
     #   httpx
     #   requests
 cffi==2.0.0
-    # via cryptography
+    # via xattr
 charset-normalizer==3.4.4
     # via requests
 cleo==2.1.0
     # via poetry
 click==8.0.3
-    # via -r tools/UdsTool/requirements.txt
+    # via -r docker/development/files/requirements.txt
 cmakelang==0.6.13
     # via -r docker/development/files/requirements.txt
+cram==0.7
+    # via -r docker/development/files/requirements.txt
 crashtest==0.4.1
     # via cleo
-cryptography==46.0.3
-    # via secretstorage
 distlib==0.4.0
     # via virtualenv
 doipclient==1.1.1
-    # via -r tools/UdsTool/requirements.txt
+    # via -r docker/development/files/requirements.txt
 dulwich==0.24.10
     # via poetry
 exceptiongroup==1.2.2
@@ -72,10 +72,6 @@ jaraco-context==6.0.1
     # via keyring
 jaraco-functools==4.3.0
     # via keyring
-jeepney==0.9.0
-    # via
-    #   keyring
-    #   secretstorage
 keyring==25.7.0
     # via poetry
 markdown-it-py==4.0.0
@@ -129,6 +125,7 @@ pytest==8.3.3
     # via -r test/pyTest/requirements.txt
 python-can==4.4.2
     # via
+    #   -r docker/development/files/requirements.txt
     #   -r test/pyTest/requirements.txt
     #   -r tools/UdsTool/requirements.txt
 pyyaml==6.0.3
@@ -144,8 +141,6 @@ requests-toolbelt==1.0.0
     # via poetry
 rich==14.2.0
     # via -r docker/development/files/requirements.txt
-secretstorage==3.5.0
-    # via keyring
 shellingham==1.5.4
     # via poetry
 six==1.17.0
@@ -161,12 +156,11 @@ trove-classifiers==2025.12.1.14
 typing-extensions==4.12.2
     # via
     #   -r test/pyTest/requirements.txt
-    #   anyio
     #   python-can
 udsoncan==1.23.1
     # via
+    #   -r docker/development/files/requirements.txt
     #   -r test/pyTest/requirements.txt
-    #   -r tools/UdsTool/requirements.txt
 urllib3==2.6.2
     # via
     #   dulwich
@@ -177,6 +171,8 @@ wrapt==1.16.0
     # via
     #   -r test/pyTest/requirements.txt
     #   python-can
+xattr==1.3.0
+    # via poetry
 zstandard==0.25.0
     # via pbs-installer
 

docker/development/files/requirements.txt

@@ -3,3 +3,10 @@ poetry~=2.2
 rich~=14.2
 sniffio~=1.3
 PyYAML~=6.0
+cram~=0.7
+can-isotp==2.0.6
+click==8.0.3
+doipclient==1.1.1
+python-can==4.4.2
+setuptools>=70.0.0
+udsoncan==1.23.1

test/setup.sh

@@ -0,0 +1,29 @@
+export TESTTMP=${PWD}
+tmux kill-server > /dev/null 2>&1
+
+declare -A OUT_POSITIONS
+OUT_POSITIONS['serial']='1'
+OUT_POSITIONS['can']='1'
+
+wait_output() { timeout 10s tail -n +${OUT_POSITIONS[$1]} -f "$TESTTMP/$1.out" | sed -u  "/$2/q" | grep "$2"; }
+send_serial() { tmux send-keys -t refApp "$1" Enter; }
+advance_output() { OUT_POSITIONS[$1]=$(cat ${TESTTMP}/$1.out | wc -l); }
+
+if [ -z "${REFAPP:-}" ]; then
+    echo "REFAPP must be set to the reference application executable path" >&2
+    return 1
+fi
+
+if [ ! -x "$REFAPP" ]; then
+    echo "Reference app not found or not executable: $REFAPP" >&2
+    return 1
+fi
+
+UDSTOOL=$(find $TESTDIR/../.. -name udsTool.py)
+candump vcan0 > "${TESTTMP}/can.out" 2>&1 &
+REFAPP_CMD="while true; do ${REFAPP}; done | sed -u 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' | sed -u 's/^[^:]*://' > ${TESTTMP}/serial.out"
+tmux new-session -d -s refApp bash -c "$REFAPP_CMD"
+
+wait_output serial 'DEBUG: Run level 9 done'
+
+export TARGET_IP="192.168.0.201"

tests/cli/can_command.t

@@ -0,0 +1,15 @@
+Setup the test enviroment
+  $ . ${TESTDIR}/../../test/setup.sh
+   RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc)
+
+Send a can frame via console command
+  $ send_serial "can send 0x100 100 2 3 46 7 20"
+
+Check that we see the frame on the bus
+  $ wait_output can 'vcan0  100'
+    vcan0  100   [6]  64 02 03 2E 07 14
+
+Same for hex bytes
+  $ send_serial "can send 0x101 0xA1 0xB2 0xC3 0xD4 0xE5 0xF6 0xA7 0xB8"
+  $ wait_output can 'vcan0  101'
+    vcan0  101   [8]  A1 B2 C3 D4 E5 F6 A7 B8

tests/cli/can_echo.t

@@ -0,0 +1,18 @@
+Setup the test enviroment
+  $ . ${TESTDIR}/../../test/setup.sh
+   RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc)
+
+Send a CAN frame to the echo service in the referenceApp
+  $ cansend vcan0 "123#AABBCCDDEEFF"
+
+Check that we see the message being sent on the bus
+  $ wait_output can 'vcan0  123'
+    vcan0  123   [6]  AA BB CC DD EE FF
+
+Check that we see notification about the reception of the message on the serial out
+  $ wait_output serial 'received CAN frame, id=0x123, length=6'
+   RefApp: CAN: DEBUG: [SocketCanTransceiver] received CAN frame, id=0x123, length=6\r (esc)
+
+Check that we see the echo response on the bus
+  $ wait_output can 'vcan0  124'
+    vcan0  124   [6]  AA BB CC DD EE FF

tests/cli/ethernet.t

@@ -0,0 +1,13 @@
+Setup the test enviroment
+  $ . ${TESTDIR}/../../test/setup.sh
+   RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc)
+
+Just send two pings and see that we do get a response
+  $ ping -c 2 -W 2 $TARGET_IP
+  PING *.*.*.* (*.*.*.*) 56(84) bytes of data. (glob)
+  64 bytes from *.*.*.*: icmp_seq=1 ttl=255 time=* ms (glob)
+  64 bytes from *.*.*.*: icmp_seq=2 ttl=255 time=* ms (glob)
+
+  --- *.*.*.* ping statistics --- (glob)
+  2 packets transmitted, 2 received, 0% packet loss, time * (glob)
+  rtt min/avg/max/mdev = */*/*/* ms (glob)

tests/cli/lc_reboot.t

@@ -0,0 +1,17 @@
+Setup the test enviroment
+  $ . ${TESTDIR}/../../test/setup.sh
+   RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc)
+
+Advance so we don't match previous output
+  $ advance_output serial
+
+Send reboot command
+  $ send_serial "lc reboot"
+
+Check that we have reached shutdown
+  $ wait_output serial 'Lifecycle shutdown complete'
+   RefApp: LIFECYCLE: INFO: Lifecycle shutdown complete\r (esc)
+
+And also check that we booted again
+  $ wait_output serial 'Run level 8 done'
+   RefApp: LIFECYCLE: DEBUG: Run level 8 done\r (esc)

tests/cli/logger.t

@@ -0,0 +1,102 @@
+Setup the test enviroment
+  $ . ${TESTDIR}/../../test/setup.sh
+   RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc)
+
+  $ advance_output serial
+  $ send_serial "logger level"
+
+Use the "logger level" command to check that it reports all components initially at "DEBUG" level
+  $ wait_output serial 'Received console command "logger level"'
+   RefApp: CONSOLE: INFO: Received console command "logger level"\r (esc)
+  $ wait_output serial 'BSP             DEBUG'
+   BSP             DEBUG\r (esc)
+  $ wait_output serial 'COMMON          DEBUG'
+   COMMON          DEBUG\r (esc)
+  $ wait_output serial 'DEMO            DEBUG'
+   DEMO            DEBUG\r (esc)
+  $ wait_output serial 'GLOBAL          DEBUG'
+   GLOBAL          DEBUG\r (esc)
+  $ wait_output serial 'LIFECYCLE       DEBUG'
+   LIFECYCLE       DEBUG\r (esc)
+  $ wait_output serial 'CONSOLE         DEBUG'
+   CONSOLE         DEBUG\r (esc)
+  $ wait_output serial 'CAN             DEBUG'
+   CAN             DEBUG\r (esc)
+  $ wait_output serial 'DOCAN           DEBUG'
+   DOCAN           DEBUG\r (esc)
+  $ wait_output serial 'UDS             DEBUG'
+   UDS             DEBUG\r (esc)
+  $ wait_output serial 'TPROUTER        DEBUG'
+   TPROUTER        DEBUG\r (esc)
+  $ wait_output serial 'ok'
+   ok\r (esc)
+  $ wait_output serial 'Console command succeeded'
+   RefApp: CONSOLE: INFO: Console command succeeded\r (esc)
+
+Set some components to different log levels
+  $ advance_output serial
+  $ send_serial "logger level BSP error"
+  $ wait_output serial 'Console command succeeded'
+   RefApp: CONSOLE: INFO: Console command succeeded\r (esc)
+
+  $ advance_output serial
+  $ send_serial "logger level COMMON info"
+  $ wait_output serial 'Console command succeeded'
+   RefApp: CONSOLE: INFO: Console command succeeded\r (esc)
+
+  $ advance_output serial
+  $ send_serial "logger level DEMO critical"
+  $ wait_output serial 'Console command succeeded'
+   RefApp: CONSOLE: INFO: Console command succeeded\r (esc)
+
+Check the level changes are reported
+  $ advance_output serial
+  $ send_serial "logger level"
+  $ wait_output serial 'BSP             ERROR'
+   BSP             ERROR\r (esc)
+  $ wait_output serial 'COMMON          INFO'
+   COMMON          INFO\r (esc)
+  $ wait_output serial 'DEMO            CRITICAL'
+   DEMO            CRITICAL\r (esc)
+  $ wait_output serial 'GLOBAL          DEBUG'
+   GLOBAL          DEBUG\r (esc)
+  $ wait_output serial 'LIFECYCLE       DEBUG'
+   LIFECYCLE       DEBUG\r (esc)
+  $ wait_output serial 'CONSOLE         DEBUG'
+   CONSOLE         DEBUG\r (esc)
+  $ wait_output serial 'CAN             DEBUG'
+   CAN             DEBUG\r (esc)
+  $ wait_output serial 'DOCAN           DEBUG'
+   DOCAN           DEBUG\r (esc)
+  $ wait_output serial 'UDS             DEBUG'
+   UDS             DEBUG\r (esc)
+  $ wait_output serial 'TPROUTER        DEBUG'
+   TPROUTER        DEBUG\r (esc)
+  $ wait_output serial 'Console command succeeded'
+   RefApp: CONSOLE: INFO: Console command succeeded\r (esc)
+
+The CAN logger should only log received frames when in DEBUG.
+Verify this by sending a frame both while in DEBUG and while in another
+level.
+  $ advance_output serial
+  $ advance_output can
+  $ cansend vcan0 "123#AABBCCDDEEFF"
+  $ wait_output can 'vcan0  123'
+    vcan0  123   [6]  AA BB CC DD EE FF
+  $ wait_output serial 'received CAN frame, id=0x123, length=6'
+   RefApp: CAN: DEBUG: [SocketCanTransceiver] received CAN frame, id=0x123, length=6\r (esc)
+  $ advance_output serial
+  $ advance_output can
+
+  $ send_serial "logger level CAN info"
+  $ wait_output serial 'Console command succeeded'
+   RefApp: CONSOLE: INFO: Console command succeeded\r (esc)
+  $ advance_output serial
+  $ advance_output can
+  $ cansend vcan0 "123#AABBCCDDEEFF"
+  $ wait_output can 'vcan0  123'
+    vcan0  123   [6]  AA BB CC DD EE FF
+  $ wait_output serial 'received CAN frame, id=0x123, length=6'
+  [1]
+  $ advance_output serial
+  $ advance_output can

tests/cli/run.sh

@@ -0,0 +1,2 @@
+
+cram --shell=/bin/bash -ivy *.t

tests/cli/storage.t

@@ -0,0 +1,47 @@
+Setup the test enviroment
+  $ . ${TESTDIR}/../../test/setup.sh
+   RefApp: LIFECYCLE: DEBUG: Run level 9 done\r (esc)
+
+This tests non-volatile storage read/write via console command.
+We write a value, reboot the target and read it back to check it survived
+a lifecycle reboot.
+This is repeated with two different data values at the same address to make
+sure we are not passing this test with a leftover written value from a previous
+test run.
+
+  $ advance_output serial
+  $ send_serial "storage write 2561 aabbccdd"
+  $ wait_output serial 'Console command succeeded'
+   RefApp: CONSOLE: INFO: Console command succeeded\r (esc)
+  $ send_serial "storage read 2561 4"
+  $ wait_output serial "aabbccdd  "
+   aabbccdd        \r (no-eol) (esc)
+  \r (esc)
+
+  $ send_serial "lc reboot"
+  $ advance_output serial
+  $ wait_output serial 'Run level 8 done'
+   RefApp: LIFECYCLE: DEBUG: Run level 8 done\r (esc)
+  $ send_serial "storage read 2561 4"
+  $ wait_output serial "aabbccdd  "
+   aabbccdd        \r (no-eol) (esc)
+  \r (esc)
+
+
+  $ advance_output serial
+  $ send_serial "storage write 2561 ddccbbaa"
+  $ wait_output serial 'Console command succeeded'
+   RefApp: CONSOLE: INFO: Console command succeeded\r (esc)
+  $ send_serial "storage read 2561 4"
+  $ wait_output serial "ddccbbaa  "
+   ddccbbaa        \r (no-eol) (esc)
+  \r (esc)
+
+  $ send_serial "lc reboot"
+  $ advance_output serial
+  $ wait_output serial 'Run level 8 done'
+   RefApp: LIFECYCLE: DEBUG: Run level 8 done\r (esc)
+  $ send_serial "storage read 2561 4"
+  $ wait_output serial "ddccbbaa  "
+   ddccbbaa        \r (no-eol) (esc)
+  \r (esc)