From bbfc6a5a74a9f96d072c6a34238f5030111ca47f Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Thu, 5 Jun 2025 09:43:25 +0000 Subject: [PATCH 01/14] Collect log using SerialConsole when SshShell fails to connect ssh may not succeed due to many reasons, at present if ssh fails it is either very difficult or impossible to triage the issue without reproducing it. This change uses SerialConsole to run commands for log collection. --- lisa/features/serial_console.py | 3 +++ lisa/node.py | 22 +++++++++++++++++++++- lisa/sut_orchestrator/azure/features.py | 16 ++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/lisa/features/serial_console.py b/lisa/features/serial_console.py index f666c3b8c0..9e6f152a14 100644 --- a/lisa/features/serial_console.py +++ b/lisa/features/serial_console.py @@ -199,3 +199,6 @@ def write(self, data: str) -> None: def close(self) -> None: # it's not required to implement close method. pass + + def execute_command(self, commands: List[str]) -> str: + raise NotImplementedError diff --git a/lisa/node.py b/lisa/node.py index 5b23b7be5e..0333d22af3 100644 --- a/lisa/node.py +++ b/lisa/node.py @@ -31,6 +31,7 @@ InitializableMixin, LisaException, RequireUserPasswordException, + TcpConnectionException, constants, fields_to_dict, generate_strong_password, @@ -535,7 +536,26 @@ def _initialize(self, *args: Any, **kwargs: Any) -> None: ) self._first_initialize = True self.log.info(f"initializing node '{self.name}' {self}") - self.shell.initialize() + try: + self.shell.initialize() + except TcpConnectionException as e: + from lisa.features import SerialConsole + + if self.features.is_supported(SerialConsole): + serial_console = self.features[SerialConsole] + + # @TODO: this is temporary, we need another way to determine the commands. + # OS is not initialized yet, so cannot use self.os. + commands = [ + "ip addr show", + "ip link show", + "systemctl status NetworkManager --no-pager --plain", + "systemctl status network --no-pager --plain", + "systemctl status systemd-networkd --no-pager --plain", + "ping -c 3 -n 8.8.8.8", + ] + serial_console.execute_command(commands=commands) + raise e self.os: OperatingSystem = OperatingSystem.create(self) self.capture_system_information("started") diff --git a/lisa/sut_orchestrator/azure/features.py b/lisa/sut_orchestrator/azure/features.py index 164ba077e7..f3825657af 100644 --- a/lisa/sut_orchestrator/azure/features.py +++ b/lisa/sut_orchestrator/azure/features.py @@ -356,6 +356,22 @@ def read(self) -> str: self._get_connection() raise e + def execute_command(self, commands: List[str]) -> str: + """ + Execute a list of commands on the serial console and return the output. + This method is used to run multiple commands in sequence. + """ + # read the serial console to clear any previous output + _ = self.read() + + # \n is required to ensure that the command is executed + for command in commands: + self.write(f"{command} \n") + + output = self.read() + + return output + def close(self) -> None: if self._ws is not None: self._log.debug("Closing connection to serial console") From f9951041b5fb13a93deeb7d1fa7fb35e6b21e896 Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Mon, 9 Jun 2025 03:12:44 +0000 Subject: [PATCH 02/14] SerialConsole login --- lisa/features/serial_console.py | 2 +- lisa/node.py | 124 ++++++++++++++++++++---- lisa/sut_orchestrator/azure/features.py | 5 +- lisa/sut_orchestrator/ready.py | 2 +- 4 files changed, 109 insertions(+), 24 deletions(-) diff --git a/lisa/features/serial_console.py b/lisa/features/serial_console.py index 9e6f152a14..18960c074e 100644 --- a/lisa/features/serial_console.py +++ b/lisa/features/serial_console.py @@ -193,7 +193,7 @@ def check_initramfs( def read(self) -> str: raise NotImplementedError - def write(self, data: str) -> None: + def write(self, data: str | List[str]) -> None: raise NotImplementedError def close(self) -> None: diff --git a/lisa/node.py b/lisa/node.py index 0333d22af3..4f539345da 100644 --- a/lisa/node.py +++ b/lisa/node.py @@ -3,6 +3,7 @@ from __future__ import annotations +import time from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath from random import randint from typing import ( @@ -536,26 +537,7 @@ def _initialize(self, *args: Any, **kwargs: Any) -> None: ) self._first_initialize = True self.log.info(f"initializing node '{self.name}' {self}") - try: - self.shell.initialize() - except TcpConnectionException as e: - from lisa.features import SerialConsole - - if self.features.is_supported(SerialConsole): - serial_console = self.features[SerialConsole] - - # @TODO: this is temporary, we need another way to determine the commands. - # OS is not initialized yet, so cannot use self.os. - commands = [ - "ip addr show", - "ip link show", - "systemctl status NetworkManager --no-pager --plain", - "systemctl status network --no-pager --plain", - "systemctl status systemd-networkd --no-pager --plain", - "ping -c 3 -n 8.8.8.8", - ] - serial_console.execute_command(commands=commands) - raise e + self.shell.initialize() self.os: OperatingSystem = OperatingSystem.create(self) self.capture_system_information("started") @@ -713,7 +695,39 @@ def set_connection_info( def _initialize(self, *args: Any, **kwargs: Any) -> None: assert self._connection_info, "call setConnectionInfo before use remote node" - super()._initialize(*args, **kwargs) + try: + super()._initialize(*args, **kwargs) + except TcpConnectionException as e: + from lisa.features import SerialConsole + + if self.features.is_supported(SerialConsole): + serial_console = self.features[SerialConsole] + + # @TODO: this is temporary, we need another way to determine the commands. + # OS is not initialized yet, so cannot use self.os. + self._reset_password() + clear = serial_console.read() + serial_console.write("\n") + serial = serial_console.read() + if "login" in serial: + serial_console.write(f"{self._connection_info.username}\n") + time.sleep(5) + serial2 = serial_console.read() + if "password" in serial2.lower(): + serial_console.write(f"{self._connection_info.password}\n") + serial3 = serial_console.read() + self.log.debug(serial3) + commands = [ + "ip addr show", + "ip link show", + "systemctl status NetworkManager --no-pager --plain", + "systemctl status network --no-pager --plain", + "systemctl status systemd-networkd --no-pager --plain", + "ping -c 3 -n 8.8.8.8", + ] + test2 = serial_console.execute_command(commands=commands) + + raise e def get_working_path(self) -> PurePath: return self._get_remote_working_path() @@ -804,6 +818,74 @@ def _check_bash_prompt(self) -> None: ssh_shell.bash_prompt = bash_prompt self.has_checked_bash_prompt = True + def _login_to_serial_console(self) -> None: + from lisa.features import SerialConsole + + if self.features.is_supported(SerialConsole): + serial_console = self.features[SerialConsole] + # clear the serial console + # write \n to serial console to get the prompt + # read the serial console output + _ = serial_console.read() + serial_console.write("\n") + serial_read = serial_console.read() + if "login" in serial_read: + password = self._get_password() + serial_console.write(f"{self._connection_info.username}\n") + password_prompt = serial_console.read() + if "password" in password_prompt.lower(): + serial_console.write(f"{password}\n") + else: + self.log.debug( + "No login prompt found, serial console is already logged in." + ) + + def _collect_info_using_serial_console(self, commands: List[str]) -> str: + from lisa.features import SerialConsole + + if self.features.is_supported(SerialConsole): + serial_console = self.features[SerialConsole] + # clear the serial console + _ = serial_console.read() + commands = [ + f"echo 'Executing: {cmd}' && {cmd}" + for cmd in [ + "ip addr show", + "ip link show", + "systemctl status NetworkManager --no-pager --plain", + "systemctl status network --no-pager --plain", + "systemctl status systemd-networkd --no-pager --plain", + "ping -c 3 -n 8.8.8.8", + ] + ] + serial_console.write(data=commands) + output = serial_console.read() + return output + else: + raise LisaException( + "SerialConsole feature is not supported, cannot collect information." + ) + + def _get_password(self, generate: bool = True) -> str: + """ + Get the password for the node. If the password is not set, it will + generate a strong password and reset it. + """ + if not self._connection_info.password: + if not generate: + raise RequireUserPasswordException( + "The password is not set, and generate is False." + ) + self.log.debug("password is not set, generating a strong password.") + if not self._reset_password(): + raise RequireUserPasswordException("Reset password failed") + password = self._connection_info.password + if not password: + raise RequireUserPasswordException( + "The password is not set, and generate is False." + ) + return password + def _reset_password(self) -> bool: from lisa.features import PasswordExtension diff --git a/lisa/sut_orchestrator/azure/features.py b/lisa/sut_orchestrator/azure/features.py index f3825657af..84a6b45f5a 100644 --- a/lisa/sut_orchestrator/azure/features.py +++ b/lisa/sut_orchestrator/azure/features.py @@ -330,9 +330,12 @@ def create_setting( return schema.FeatureSettings.create(cls.name()) @retry(tries=3, delay=5) - def write(self, data: str) -> None: + def write(self, data: str | List[str]) -> None: # websocket connection is not stable, so we need to retry try: + if isinstance(data, list): + # if data is a list, join it with \n and add a newline to the last item + data = "\n".join(data) + "\n" self._write(data) return except websockets.ConnectionClosed as e: # type: ignore diff --git a/lisa/sut_orchestrator/ready.py b/lisa/sut_orchestrator/ready.py index 71093b21a1..67b8e8fc3a 100644 --- a/lisa/sut_orchestrator/ready.py +++ b/lisa/sut_orchestrator/ready.py @@ -100,5 +100,5 @@ def _get_console_log(self, saved_path: Optional[Path]) -> bytes: def read(self) -> str: return "" - def write(self, data: str) -> None: + def write(self, data: str | List[str]) -> None: pass From c063261240edf80a1e6230d5c9ca17a7e6c60355 Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Mon, 9 Jun 2025 12:51:59 +0000 Subject: [PATCH 03/14] Add RunCommand feature in Azure and use it to collect logs --- lisa/features/__init__.py | 2 + lisa/node.py | 50 +++++++++++------------- lisa/sut_orchestrator/azure/features.py | 47 ++++++++++++++++++++++ lisa/sut_orchestrator/azure/platform_.py | 1 + 4 files changed, 72 insertions(+), 28 deletions(-) diff --git a/lisa/features/__init__.py b/lisa/features/__init__.py index 38da40120b..eaabf00662 100644 --- a/lisa/features/__init__.py +++ b/lisa/features/__init__.py @@ -26,6 +26,7 @@ from .nvme import Nvme, NvmeSettings from .password_extension import PasswordExtension from .resize import Resize, ResizeAction +from .run_command import RunCommand from .security_profile import ( SecureBootEnabled, SecurityProfile, @@ -64,6 +65,7 @@ "PasswordExtension", "Resize", "ResizeAction", + "RunCommand", "SecureBootEnabled", "SecurityProfile", "SecurityProfileSettings", diff --git a/lisa/node.py b/lisa/node.py index 4f539345da..b2debcfa7b 100644 --- a/lisa/node.py +++ b/lisa/node.py @@ -698,34 +698,28 @@ def _initialize(self, *args: Any, **kwargs: Any) -> None: try: super()._initialize(*args, **kwargs) except TcpConnectionException as e: - from lisa.features import SerialConsole - - if self.features.is_supported(SerialConsole): - serial_console = self.features[SerialConsole] - - # @TODO: this is temporary, we need another way to determine the commands. - # OS is not initialized yet, so cannot use self.os. - self._reset_password() - clear = serial_console.read() - serial_console.write("\n") - serial = serial_console.read() - if "login" in serial: - serial_console.write(f"{self._connection_info.username}\n") - time.sleep(5) - serial2 = serial_console.read() - if "password" in serial2.lower(): - serial_console.write(f"{self._connection_info.password}\n") - serial3 = serial_console.read() - self.log.debug(serial3) - commands = [ - "ip addr show", - "ip link show", - "systemctl status NetworkManager --no-pager --plain", - "systemctl status network --no-pager --plain", - "systemctl status systemd-networkd --no-pager --plain", - "ping -c 3 -n 8.8.8.8", - ] - test2 = serial_console.execute_command(commands=commands) + from lisa.features.run_command import RunCommand + + run_command = self.features[RunCommand] + commands = [ + "echo 'Executing: ip addr show'", + "ip addr show", + "echo 'Executing: ip link show'", + "ip link show", + "echo 'Executing: systemctl status NetworkManager --no-pager --plain'", + "systemctl status NetworkManager --no-pager --plain", + "echo 'Executing: systemctl status network --no-pager --plain'", + "systemctl status network --no-pager --plain", + "echo 'Executing: systemctl status systemd-networkd --no-pager --plain'", + "systemctl status systemd-networkd --no-pager --plain", + "echo 'Executing: ping -c 3 -n 8.8.8.8'", + "ping -c 3 -n 8.8.8.8", + ] + out = run_command.execute(commands=commands) + self.log.info(f"Collected information using run_command:\n{out}") + self._login_to_serial_console() + output = self._collect_info_using_serial_console(commands=commands) + self.log.info(f"Collected information using serial console:\n{output}") raise e diff --git a/lisa/sut_orchestrator/azure/features.py b/lisa/sut_orchestrator/azure/features.py index 84a6b45f5a..4c6405bb9b 100644 --- a/lisa/sut_orchestrator/azure/features.py +++ b/lisa/sut_orchestrator/azure/features.py @@ -6,6 +6,7 @@ import re import string import time +import uuid from dataclasses import dataclass, field from functools import partial from pathlib import Path @@ -26,6 +27,8 @@ DiskCreateOptionTypes, HardwareProfile, NetworkInterfaceReference, + RunCommandInput, + RunCommandInputParameter, VirtualMachineExtension, VirtualMachineUpdate, ) @@ -3760,3 +3763,47 @@ def _prepare_azure_file_share( sudo=True, append=True, ) + + +class RunCommand(AzureFeatureMixin, features.RunCommand): + + @classmethod + def create_setting( + cls, *args: Any, **kwargs: Any + ) -> Optional[schema.FeatureSettings]: + return schema.FeatureSettings.create(cls.name()) + + @classmethod + def can_disable(cls) -> bool: + return False + + def is_enabled(self) -> bool: + # RunCommand is always enabled for Azure + return True + + def execute(self, commands: List[str]) -> str: + """ + Executes a list of commands on the Azure VM using RunCommand. + + :param commands: A list of shell commands to execute. + :return: The output of the commands. + """ + context = get_node_context(self._node) + platform = self._platform + compute_client = get_compute_client(platform) + + # Prepare the RunCommandInput for Azure + command = RunCommandInput( + command_id="RunShellScript", + script=commands, + ) + + # Execute the command on the VM + operation = compute_client.virtual_machines.begin_run_command( + resource_group_name=context.resource_group_name, + vm_name=context.vm_name, + parameters=command, + ) + result = wait_operation(operation=operation, failure_identity="run command") + + return result["value"][0]["message"] diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index da1e552b7f..90add78952 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -501,6 +501,7 @@ def supported_features(cls) -> List[Type[feature.Feature]]: features.Availability, features.Infiniband, features.Hibernation, + features.RunCommand, ] def _prepare_environment(self, environment: Environment, log: Logger) -> bool: From a1ca77bab982a4b7b14e433dc206fff8a8a65a66 Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Mon, 7 Jul 2025 09:48:01 +0000 Subject: [PATCH 04/14] Revert changes - to be squashed at the end --- lisa/features/serial_console.py | 5 +---- lisa/sut_orchestrator/ready.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lisa/features/serial_console.py b/lisa/features/serial_console.py index 18960c074e..f666c3b8c0 100644 --- a/lisa/features/serial_console.py +++ b/lisa/features/serial_console.py @@ -193,12 +193,9 @@ def check_initramfs( def read(self) -> str: raise NotImplementedError - def write(self, data: str | List[str]) -> None: + def write(self, data: str) -> None: raise NotImplementedError def close(self) -> None: # it's not required to implement close method. pass - - def execute_command(self, commands: List[str]) -> str: - raise NotImplementedError diff --git a/lisa/sut_orchestrator/ready.py b/lisa/sut_orchestrator/ready.py index 67b8e8fc3a..71093b21a1 100644 --- a/lisa/sut_orchestrator/ready.py +++ b/lisa/sut_orchestrator/ready.py @@ -100,5 +100,5 @@ def _get_console_log(self, saved_path: Optional[Path]) -> bytes: def read(self) -> str: return "" - def write(self, data: str | List[str]) -> None: + def write(self, data: str) -> None: pass From cbf989434f332ec615dfb4d55ac65b1b585b57ea Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Mon, 7 Jul 2025 09:56:53 +0000 Subject: [PATCH 05/14] Remove serial console log collection for network failure --- lisa/node.py | 79 ++++++++++--------------- lisa/sut_orchestrator/azure/features.py | 21 +------ 2 files changed, 31 insertions(+), 69 deletions(-) diff --git a/lisa/node.py b/lisa/node.py index b2debcfa7b..162aa1a978 100644 --- a/lisa/node.py +++ b/lisa/node.py @@ -698,29 +698,9 @@ def _initialize(self, *args: Any, **kwargs: Any) -> None: try: super()._initialize(*args, **kwargs) except TcpConnectionException as e: - from lisa.features.run_command import RunCommand - - run_command = self.features[RunCommand] - commands = [ - "echo 'Executing: ip addr show'", - "ip addr show", - "echo 'Executing: ip link show'", - "ip link show", - "echo 'Executing: systemctl status NetworkManager --no-pager --plain'", - "systemctl status NetworkManager --no-pager --plain", - "echo 'Executing: systemctl status network --no-pager --plain'", - "systemctl status network --no-pager --plain", - "echo 'Executing: systemctl status systemd-networkd --no-pager --plain'", - "systemctl status systemd-networkd --no-pager --plain", - "echo 'Executing: ping -c 3 -n 8.8.8.8'", - "ping -c 3 -n 8.8.8.8", - ] - out = run_command.execute(commands=commands) - self.log.info(f"Collected information using run_command:\n{out}") - self._login_to_serial_console() - output = self._collect_info_using_serial_console(commands=commands) - self.log.info(f"Collected information using serial console:\n{output}") - + vm_logs = self._collect_logs_using_platform() + if vm_logs: + self.log.info(f"Collected information using platform:\n{vm_logs}") raise e def get_working_path(self) -> PurePath: @@ -765,6 +745,33 @@ def check_sudo_password_required(self) -> None: raise RequireUserPasswordException("Reset password failed") self._check_password_and_store_prompt() + def _collect_logs_using_platform(self) -> Optional[str]: + """ + Collects information using the RunCommand feature. + This is used when the connection to the node is not stable. + """ + from lisa.features import RunCommand + + if self.features.is_supported(RunCommand): + run_command = self.features[RunCommand] + commands = [ + "echo 'Executing: ip addr show'", + "ip addr show", + "echo 'Executing: ip link show'", + "ip link show", + "echo 'Executing: systemctl status NetworkManager --no-pager --plain'", + "systemctl status NetworkManager --no-pager --plain", + "echo 'Executing: systemctl status network --no-pager --plain'", + "systemctl status network --no-pager --plain", + "echo 'Executing: systemctl status systemd-networkd --no-pager --plain'", + "systemctl status systemd-networkd --no-pager --plain", + "echo 'Executing: ping -c 3 -n 8.8.8.8'", + "ping -c 3 -n 8.8.8.8", + ] + out = run_command.execute(commands=commands) + return out + return None + def _check_password_and_store_prompt(self) -> None: # self.shell.is_sudo_required_password is true, so running sudo command # will input password in process.wait_result. Check running sudo again @@ -834,32 +841,6 @@ def _login_to_serial_console(self) -> None: "No login prompt found, serial console is already logged in." ) - def _collect_info_using_serial_console(self, commands: List[str]) -> str: - from lisa.features import SerialConsole - - if self.features.is_supported(SerialConsole): - serial_console = self.features[SerialConsole] - # clear the serial console - _ = serial_console.read() - commands = [ - f"echo 'Executing: {cmd}' && {cmd}" - for cmd in [ - "ip addr show", - "ip link show", - "systemctl status NetworkManager --no-pager --plain", - "systemctl status network --no-pager --plain", - "systemctl status systemd-networkd --no-pager --plain", - "ping -c 3 -n 8.8.8.8", - ] - ] - serial_console.write(data=commands) - output = serial_console.read() - return output - else: - raise LisaException( - "SerialConsole feature is not supported, cannot collect information." - ) - def _get_password(self, generate: bool = True) -> str: """ Get the password for the node. If the password is not set, it will diff --git a/lisa/sut_orchestrator/azure/features.py b/lisa/sut_orchestrator/azure/features.py index 4c6405bb9b..ded63e9afb 100644 --- a/lisa/sut_orchestrator/azure/features.py +++ b/lisa/sut_orchestrator/azure/features.py @@ -333,12 +333,9 @@ def create_setting( return schema.FeatureSettings.create(cls.name()) @retry(tries=3, delay=5) - def write(self, data: str | List[str]) -> None: + def write(self, data: str) -> None: # websocket connection is not stable, so we need to retry try: - if isinstance(data, list): - # if data is a list, join it with \n and add a newline to the last item - data = "\n".join(data) + "\n" self._write(data) return except websockets.ConnectionClosed as e: # type: ignore @@ -362,22 +359,6 @@ def read(self) -> str: self._get_connection() raise e - def execute_command(self, commands: List[str]) -> str: - """ - Execute a list of commands on the serial console and return the output. - This method is used to run multiple commands in sequence. - """ - # read the serial console to clear any previous output - _ = self.read() - - # \n is required to ensure that the command is executed - for command in commands: - self.write(f"{command} \n") - - output = self.read() - - return output - def close(self) -> None: if self._ws is not None: self._log.debug("Closing connection to serial console") From 79355055bf399f5bf6ed903ccd86f7b1284e2109 Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Tue, 8 Jul 2025 06:43:54 +0000 Subject: [PATCH 06/14] Add RunCommand class --- lisa/features/run_command.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 lisa/features/run_command.py diff --git a/lisa/features/run_command.py b/lisa/features/run_command.py new file mode 100644 index 0000000000..669b1f7f25 --- /dev/null +++ b/lisa/features/run_command.py @@ -0,0 +1,22 @@ +from typing import List + +from lisa.feature import Feature + +FEATURE_NAME_RUNCOMMAND = "RunCommand" + + +class RunCommand(Feature): + @classmethod + def name(cls) -> str: + return FEATURE_NAME_RUNCOMMAND + + def execute(self, commands: List[str]) -> str: + """ + Executes a list of commands on the node and returns their outputs. + + :param commands: A list of shell commands to execute. + :return: A list of outputs corresponding to each command. + """ + raise NotImplementedError( + "The execute method must be implemented by the subclass." + ) From 82ed779832ee205c986727305ae5329472d310f4 Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Tue, 8 Jul 2025 09:09:56 +0000 Subject: [PATCH 07/14] Changes based on review commands Introduce NonSshExecutor. It Internally uses SerialConsole/RunCommand --- lisa/features/__init__.py | 4 +- lisa/features/non_ssh_executor.py | 53 +++++++++++++++++ lisa/features/run_command.py | 22 -------- lisa/features/serial_console.py | 29 ++++++++++ lisa/node.py | 72 ++++++++---------------- lisa/sut_orchestrator/azure/features.py | 29 ++++++++-- lisa/sut_orchestrator/azure/platform_.py | 1 + 7 files changed, 134 insertions(+), 76 deletions(-) create mode 100644 lisa/features/non_ssh_executor.py delete mode 100644 lisa/features/run_command.py diff --git a/lisa/features/__init__.py b/lisa/features/__init__.py index eaabf00662..f63ce65332 100644 --- a/lisa/features/__init__.py +++ b/lisa/features/__init__.py @@ -23,10 +23,10 @@ from .nested_virtualization import NestedVirtualization from .network_interface import NetworkInterface, Sriov, Synthetic from .nfs import Nfs +from .non_ssh_executor import NonSshExecutor from .nvme import Nvme, NvmeSettings from .password_extension import PasswordExtension from .resize import Resize, ResizeAction -from .run_command import RunCommand from .security_profile import ( SecureBootEnabled, SecurityProfile, @@ -65,7 +65,6 @@ "PasswordExtension", "Resize", "ResizeAction", - "RunCommand", "SecureBootEnabled", "SecurityProfile", "SecurityProfileSettings", @@ -75,4 +74,5 @@ "VMStatus", "Synthetic", "StartStop", + "NonSshExecutor", ] diff --git a/lisa/features/non_ssh_executor.py b/lisa/features/non_ssh_executor.py new file mode 100644 index 0000000000..a287454e7f --- /dev/null +++ b/lisa/features/non_ssh_executor.py @@ -0,0 +1,53 @@ +from lisa.feature import Feature +from lisa.features.serial_console import SerialConsole + + +class NonSshExecutor(Feature): + """ + NonSshExecutor is used to run commands on the node when SSH is not available. + Lisa by default uses SSH for connection, but this feature provides an alternative + execution method for scenarios where SSH connectivity is not possible or desired. + """ + + COMMANDS_TO_EXECUTE = [ + "ip addr show", + "ip link show", + "systemctl status NetworkManager --no-pager --plain", + "systemctl status network --no-pager --plain", + "systemctl status systemd-networkd --no-pager --plain", + "ping -c 3 -n 8.8.8.8", + ] + + @classmethod + def name(cls) -> str: + return "NonSshExecutor" + + def enabled(self) -> bool: + return True + + def execute(self, commands: list[str] = COMMANDS_TO_EXECUTE) -> list[str]: + """ + Executes a list of commands on the node and returns their outputs. + + :param commands: A list of shell commands to execute. + :return: A string containing the output of the executed commands. + """ + out = [] + serial_console = self._node.features[SerialConsole] + serial_console.login() + # clear the console before executing commands + serial_console.write("\n") + _ = serial_console.read() + for command in commands: + serial_console.write(self._add_newline(command)) + out.append(serial_console.read()) + return out + + def _add_newline(self, command: str) -> str: + """ + Adds a newline character to the command if it does not already end with one. + newline is required to run the command in serial console. + """ + if not command.endswith("\n"): + return f"{command}\n" + return command diff --git a/lisa/features/run_command.py b/lisa/features/run_command.py deleted file mode 100644 index 669b1f7f25..0000000000 --- a/lisa/features/run_command.py +++ /dev/null @@ -1,22 +0,0 @@ -from typing import List - -from lisa.feature import Feature - -FEATURE_NAME_RUNCOMMAND = "RunCommand" - - -class RunCommand(Feature): - @classmethod - def name(cls) -> str: - return FEATURE_NAME_RUNCOMMAND - - def execute(self, commands: List[str]) -> str: - """ - Executes a list of commands on the node and returns their outputs. - - :param commands: A list of shell commands to execute. - :return: A list of outputs corresponding to each command. - """ - raise NotImplementedError( - "The execute method must be implemented by the subclass." - ) diff --git a/lisa/features/serial_console.py b/lisa/features/serial_console.py index f666c3b8c0..ba2cc2e0a8 100644 --- a/lisa/features/serial_console.py +++ b/lisa/features/serial_console.py @@ -13,6 +13,7 @@ get_datetime_path, get_matched_str, ) +from lisa.util.constants import ENVIRONMENTS_NODES_REMOTE_USERNAME FEATURE_NAME_SERIAL_CONSOLE = "SerialConsole" NAME_SERIAL_CONSOLE_LOG = "serial_console.log" @@ -190,6 +191,34 @@ def check_initramfs( f"{initramfs_logs} {filesystem_exception_logs}" ) + def login(self) -> None: + # Clear the serial console and try to get the login prompt + self.read() + self.write("\n") + serial_output = self.read() + + if "login" not in serial_output: + self._log.debug( + "No login prompt found, serial console is already logged in." + ) + return + + from lisa.node import RemoteNode + + if not isinstance(self._node, RemoteNode): + raise LisaException( + "SerialConsole login is only implemented for RemoteNode" + ) + + username = self._node.connection_info[ENVIRONMENTS_NODES_REMOTE_USERNAME] + password = self._node.get_password() + + self.write(f"{username}\n") + password_prompt = self.read() + + if "password" in password_prompt.lower(): + self.write(f"{password}\n") + def read(self) -> str: raise NotImplementedError diff --git a/lisa/node.py b/lisa/node.py index 162aa1a978..34ad7c64f8 100644 --- a/lisa/node.py +++ b/lisa/node.py @@ -698,9 +698,16 @@ def _initialize(self, *args: Any, **kwargs: Any) -> None: try: super()._initialize(*args, **kwargs) except TcpConnectionException as e: - vm_logs = self._collect_logs_using_platform() - if vm_logs: - self.log.info(f"Collected information using platform:\n{vm_logs}") + try: + vm_logs = self._collect_logs_using_non_ssh_executor() + if vm_logs: + self.log.info( + f"Collected information using non-ssh executor:\n{vm_logs}" + ) + except Exception as log_error: + self.log.debug( + f"Failed to collect logs using non-ssh executor: {log_error}" + ) raise e def get_working_path(self) -> PurePath: @@ -745,31 +752,22 @@ def check_sudo_password_required(self) -> None: raise RequireUserPasswordException("Reset password failed") self._check_password_and_store_prompt() - def _collect_logs_using_platform(self) -> Optional[str]: + def _collect_logs_using_non_ssh_executor(self) -> Optional[str]: """ - Collects information using the RunCommand feature. + Collects information using the NonSshExecutor feature. This is used when the connection to the node is not stable. """ - from lisa.features import RunCommand - - if self.features.is_supported(RunCommand): - run_command = self.features[RunCommand] - commands = [ - "echo 'Executing: ip addr show'", - "ip addr show", - "echo 'Executing: ip link show'", - "ip link show", - "echo 'Executing: systemctl status NetworkManager --no-pager --plain'", - "systemctl status NetworkManager --no-pager --plain", - "echo 'Executing: systemctl status network --no-pager --plain'", - "systemctl status network --no-pager --plain", - "echo 'Executing: systemctl status systemd-networkd --no-pager --plain'", - "systemctl status systemd-networkd --no-pager --plain", - "echo 'Executing: ping -c 3 -n 8.8.8.8'", - "ping -c 3 -n 8.8.8.8", - ] - out = run_command.execute(commands=commands) - return out + from lisa.features import NonSshExecutor + + if self.features.is_supported(NonSshExecutor): + non_ssh_executor = self.features[NonSshExecutor] + out = non_ssh_executor.execute() + return "\n".join(out) + else: + self.log.debug( + f"NonSshExecutor is not supported on {self.name}, " + "cannot collect logs using non-ssh executor." + ) return None def _check_password_and_store_prompt(self) -> None: @@ -819,29 +817,7 @@ def _check_bash_prompt(self) -> None: ssh_shell.bash_prompt = bash_prompt self.has_checked_bash_prompt = True - def _login_to_serial_console(self) -> None: - from lisa.features import SerialConsole - - if self.features.is_supported(SerialConsole): - serial_console = self.features[SerialConsole] - # clear the serial console - # write \n to serial console to get the prompt - # read the serial console output - _ = serial_console.read() - serial_console.write("\n") - serial_read = serial_console.read() - if "login" in serial_read: - password = self._get_password() - serial_console.write(f"{self._connection_info.username}\n") - password_prompt = serial_console.read() - if "password" in password_prompt.lower(): - serial_console.write(f"{password}\n") - else: - self.log.debug( - "No login prompt found, serial console is already logged in." - ) - - def _get_password(self, generate: bool = True) -> str: + def get_password(self, generate: bool = True) -> str: """ Get the password for the node. If the password is not set, it will generate a strong password and reset it. diff --git a/lisa/sut_orchestrator/azure/features.py b/lisa/sut_orchestrator/azure/features.py index ded63e9afb..8d520a6269 100644 --- a/lisa/sut_orchestrator/azure/features.py +++ b/lisa/sut_orchestrator/azure/features.py @@ -3746,7 +3746,7 @@ def _prepare_azure_file_share( ) -class RunCommand(AzureFeatureMixin, features.RunCommand): +class RunCommand(AzureFeatureMixin, Feature): @classmethod def create_setting( @@ -3774,17 +3774,38 @@ def execute(self, commands: List[str]) -> str: compute_client = get_compute_client(platform) # Prepare the RunCommandInput for Azure - command = RunCommandInput( + run_command_input = RunCommandInput( command_id="RunShellScript", - script=commands, + script=self._add_echo_before_command(commands), ) # Execute the command on the VM operation = compute_client.virtual_machines.begin_run_command( resource_group_name=context.resource_group_name, vm_name=context.vm_name, - parameters=command, + parameters=run_command_input, ) result = wait_operation(operation=operation, failure_identity="run command") return result["value"][0]["message"] + + def _add_echo_before_command(self, commands: List[str]): + """ + Adds an echo command before each command in the list to ensure + that the output of each command is captured in the logs. + """ + return [f"echo 'Running command: {cmd}' && {cmd}" for cmd in commands] + + +class NonSshExecutor(AzureFeatureMixin, features.NonSshExecutor): + def execute( + self, commands: list[str] = features.NonSshExecutor.COMMANDS_TO_EXECUTE + ) -> list[str]: + # RunCommand is faster than SerialConsole. Hence attempt to use it first. + try: + output = self._node.features[RunCommand].execute(commands) + return [output] + except Exception as e: + self._log.info(f"RunCommand failed: {e}") + # Fallback to the default non-SSH executor behavior + return super().execute(commands) diff --git a/lisa/sut_orchestrator/azure/platform_.py b/lisa/sut_orchestrator/azure/platform_.py index 90add78952..3adb583e18 100644 --- a/lisa/sut_orchestrator/azure/platform_.py +++ b/lisa/sut_orchestrator/azure/platform_.py @@ -502,6 +502,7 @@ def supported_features(cls) -> List[Type[feature.Feature]]: features.Infiniband, features.Hibernation, features.RunCommand, + features.NonSshExecutor, ] def _prepare_environment(self, environment: Environment, log: Logger) -> bool: From 74c92e89c99424fe91d18cc679f439881dc3e5fe Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Mon, 14 Jul 2025 09:55:52 +0000 Subject: [PATCH 08/14] Review comments --- lisa/features/non_ssh_executor.py | 19 +++++++-------- lisa/features/serial_console.py | 2 +- lisa/node.py | 32 ++++++++++++++----------- lisa/sut_orchestrator/azure/features.py | 9 ++----- 4 files changed, 29 insertions(+), 33 deletions(-) diff --git a/lisa/features/non_ssh_executor.py b/lisa/features/non_ssh_executor.py index a287454e7f..6f9c8d11d7 100644 --- a/lisa/features/non_ssh_executor.py +++ b/lisa/features/non_ssh_executor.py @@ -1,3 +1,5 @@ +from typing import List + from lisa.feature import Feature from lisa.features.serial_console import SerialConsole @@ -9,15 +11,6 @@ class NonSshExecutor(Feature): execution method for scenarios where SSH connectivity is not possible or desired. """ - COMMANDS_TO_EXECUTE = [ - "ip addr show", - "ip link show", - "systemctl status NetworkManager --no-pager --plain", - "systemctl status network --no-pager --plain", - "systemctl status systemd-networkd --no-pager --plain", - "ping -c 3 -n 8.8.8.8", - ] - @classmethod def name(cls) -> str: return "NonSshExecutor" @@ -25,7 +18,7 @@ def name(cls) -> str: def enabled(self) -> bool: return True - def execute(self, commands: list[str] = COMMANDS_TO_EXECUTE) -> list[str]: + def execute(self, commands: List[str]) -> List[str]: """ Executes a list of commands on the node and returns their outputs. @@ -33,8 +26,12 @@ def execute(self, commands: list[str] = COMMANDS_TO_EXECUTE) -> list[str]: :return: A string containing the output of the executed commands. """ out = [] + if not self._node.features.is_supported(SerialConsole): + raise NotImplementedError( + "NonSshExecutor requires SerialConsole feature to be supported." + ) serial_console = self._node.features[SerialConsole] - serial_console.login() + serial_console.ensure_login() # clear the console before executing commands serial_console.write("\n") _ = serial_console.read() diff --git a/lisa/features/serial_console.py b/lisa/features/serial_console.py index ba2cc2e0a8..37abafd488 100644 --- a/lisa/features/serial_console.py +++ b/lisa/features/serial_console.py @@ -191,7 +191,7 @@ def check_initramfs( f"{initramfs_logs} {filesystem_exception_logs}" ) - def login(self) -> None: + def ensure_login(self) -> None: # Clear the serial console and try to get the login prompt self.read() self.write("\n") diff --git a/lisa/node.py b/lisa/node.py index 34ad7c64f8..269ea0f700 100644 --- a/lisa/node.py +++ b/lisa/node.py @@ -3,7 +3,6 @@ from __future__ import annotations -import time from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath from random import randint from typing import ( @@ -697,18 +696,14 @@ def _initialize(self, *args: Any, **kwargs: Any) -> None: assert self._connection_info, "call setConnectionInfo before use remote node" try: super()._initialize(*args, **kwargs) - except TcpConnectionException as e: + except TcpConnectionException: try: - vm_logs = self._collect_logs_using_non_ssh_executor() - if vm_logs: - self.log.info( - f"Collected information using non-ssh executor:\n{vm_logs}" - ) + self._collect_logs_using_non_ssh_executor() except Exception as log_error: self.log.debug( f"Failed to collect logs using non-ssh executor: {log_error}" ) - raise e + raise def get_working_path(self) -> PurePath: return self._get_remote_working_path() @@ -752,23 +747,32 @@ def check_sudo_password_required(self) -> None: raise RequireUserPasswordException("Reset password failed") self._check_password_and_store_prompt() - def _collect_logs_using_non_ssh_executor(self) -> Optional[str]: + def _collect_logs_using_non_ssh_executor(self): """ Collects information using the NonSshExecutor feature. This is used when the connection to the node is not stable. """ from lisa.features import NonSshExecutor + commands = [ + "ip addr show", + "ip link show", + "systemctl status NetworkManager --no-pager --plain", + "systemctl status network --no-pager --plain", + "systemctl status systemd-networkd --no-pager --plain", + "ping -c 3 -n 8.8.8.8", + ] + if self.features.is_supported(NonSshExecutor): non_ssh_executor = self.features[NonSshExecutor] - out = non_ssh_executor.execute() - return "\n".join(out) + out = non_ssh_executor.execute(commands=commands) + out = "\n\n".join(out) + self.log.info(f"Collected information using NonSshExecutor:\n{out}") else: self.log.debug( f"NonSshExecutor is not supported on {self.name}, " "cannot collect logs using non-ssh executor." ) - return None def _check_password_and_store_prompt(self) -> None: # self.shell.is_sudo_required_password is true, so running sudo command @@ -825,7 +829,7 @@ def get_password(self, generate: bool = True) -> str: if not self._connection_info.password: if not generate: raise RequireUserPasswordException( - "The password is not set, and generate is False." + "The password is not set and generation is disabled." ) self.log.debug("password is not set, generating a strong password.") if not self._reset_password(): @@ -833,7 +837,7 @@ def get_password(self, generate: bool = True) -> str: password = self._connection_info.password if not password: raise RequireUserPasswordException( - "The password is not set, and generate is False." + "The password has neither been set nor generated." ) return password diff --git a/lisa/sut_orchestrator/azure/features.py b/lisa/sut_orchestrator/azure/features.py index 8d520a6269..852106935f 100644 --- a/lisa/sut_orchestrator/azure/features.py +++ b/lisa/sut_orchestrator/azure/features.py @@ -6,7 +6,6 @@ import re import string import time -import uuid from dataclasses import dataclass, field from functools import partial from pathlib import Path @@ -28,7 +27,6 @@ HardwareProfile, NetworkInterfaceReference, RunCommandInput, - RunCommandInputParameter, VirtualMachineExtension, VirtualMachineUpdate, ) @@ -3747,7 +3745,6 @@ def _prepare_azure_file_share( class RunCommand(AzureFeatureMixin, Feature): - @classmethod def create_setting( cls, *args: Any, **kwargs: Any @@ -3789,7 +3786,7 @@ def execute(self, commands: List[str]) -> str: return result["value"][0]["message"] - def _add_echo_before_command(self, commands: List[str]): + def _add_echo_before_command(self, commands: List[str]) -> List[str]: """ Adds an echo command before each command in the list to ensure that the output of each command is captured in the logs. @@ -3798,9 +3795,7 @@ def _add_echo_before_command(self, commands: List[str]): class NonSshExecutor(AzureFeatureMixin, features.NonSshExecutor): - def execute( - self, commands: list[str] = features.NonSshExecutor.COMMANDS_TO_EXECUTE - ) -> list[str]: + def execute(self, commands: List[str]) -> List[str]: # RunCommand is faster than SerialConsole. Hence attempt to use it first. try: output = self._node.features[RunCommand].execute(commands) From 4809777318e73dfc156e2ef72e6503773bc4b2cf Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Mon, 14 Jul 2025 15:26:12 +0000 Subject: [PATCH 09/14] Update commands to execute --- lisa/node.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lisa/node.py b/lisa/node.py index 269ea0f700..8d3b672f70 100644 --- a/lisa/node.py +++ b/lisa/node.py @@ -757,10 +757,13 @@ def _collect_logs_using_non_ssh_executor(self): commands = [ "ip addr show", "ip link show", + "ip neigh", "systemctl status NetworkManager --no-pager --plain", "systemctl status network --no-pager --plain", "systemctl status systemd-networkd --no-pager --plain", "ping -c 3 -n 8.8.8.8", + "cat /var/log/waagent.log | tail -n 50", + "journalctl -n 50 --no-pager", ] if self.features.is_supported(NonSshExecutor): From 40957a3f6d233f8df9902a91330234cbc07b8e29 Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Tue, 15 Jul 2025 05:58:07 +0000 Subject: [PATCH 10/14] Serial Console: check for prompt before executing --- lisa/features/non_ssh_executor.py | 9 +++++++-- lisa/node.py | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lisa/features/non_ssh_executor.py b/lisa/features/non_ssh_executor.py index 6f9c8d11d7..42b1360d2e 100644 --- a/lisa/features/non_ssh_executor.py +++ b/lisa/features/non_ssh_executor.py @@ -2,6 +2,7 @@ from lisa.feature import Feature from lisa.features.serial_console import SerialConsole +from lisa.util import LisaException class NonSshExecutor(Feature): @@ -25,7 +26,7 @@ def execute(self, commands: List[str]) -> List[str]: :param commands: A list of shell commands to execute. :return: A string containing the output of the executed commands. """ - out = [] + out: List[str] = [] if not self._node.features.is_supported(SerialConsole): raise NotImplementedError( "NonSshExecutor requires SerialConsole feature to be supported." @@ -33,8 +34,12 @@ def execute(self, commands: List[str]) -> List[str]: serial_console = self._node.features[SerialConsole] serial_console.ensure_login() # clear the console before executing commands - serial_console.write("\n") _ = serial_console.read() + # write a newline and read to make sure serial console has the prompt + serial_console.write("\n") + response = serial_console.read() + if not response or "$" not in response and "#" not in response: + raise LisaException("Serial console prompt not found in output") for command in commands: serial_console.write(self._add_newline(command)) out.append(serial_console.read()) diff --git a/lisa/node.py b/lisa/node.py index 8d3b672f70..ac1bc5a547 100644 --- a/lisa/node.py +++ b/lisa/node.py @@ -758,12 +758,12 @@ def _collect_logs_using_non_ssh_executor(self): "ip addr show", "ip link show", "ip neigh", - "systemctl status NetworkManager --no-pager --plain", - "systemctl status network --no-pager --plain", - "systemctl status systemd-networkd --no-pager --plain", "ping -c 3 -n 8.8.8.8", "cat /var/log/waagent.log | tail -n 50", "journalctl -n 50 --no-pager", + "systemctl status NetworkManager --no-pager --plain", + "systemctl status network --no-pager --plain", + "systemctl status systemd-networkd --no-pager --plain", ] if self.features.is_supported(NonSshExecutor): From 372506cdab79fb1cfd04bb73e23c77651bef2070 Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Tue, 15 Jul 2025 07:11:47 +0000 Subject: [PATCH 11/14] Fix connection alive issue --- lisa/features/non_ssh_executor.py | 34 ++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/lisa/features/non_ssh_executor.py b/lisa/features/non_ssh_executor.py index 42b1360d2e..0c413a6393 100644 --- a/lisa/features/non_ssh_executor.py +++ b/lisa/features/non_ssh_executor.py @@ -26,23 +26,33 @@ def execute(self, commands: List[str]) -> List[str]: :param commands: A list of shell commands to execute. :return: A string containing the output of the executed commands. """ - out: List[str] = [] + if not self._node.features.is_supported(SerialConsole): raise NotImplementedError( "NonSshExecutor requires SerialConsole feature to be supported." ) + out = self._execute(commands) + return out + + def _execute(self, commands: List[str]) -> List[str]: + out: List[str] = [] serial_console = self._node.features[SerialConsole] - serial_console.ensure_login() - # clear the console before executing commands - _ = serial_console.read() - # write a newline and read to make sure serial console has the prompt - serial_console.write("\n") - response = serial_console.read() - if not response or "$" not in response and "#" not in response: - raise LisaException("Serial console prompt not found in output") - for command in commands: - serial_console.write(self._add_newline(command)) - out.append(serial_console.read()) + try: + serial_console.ensure_login() + # clear the console before executing commands + _ = serial_console.read() + # write a newline and read to make sure serial console has the prompt + serial_console.write("\n") + response = serial_console.read() + if not response or "$" not in response and "#" not in response: + raise LisaException("Serial console prompt not found in output") + for command in commands: + serial_console.write(self._add_newline(command)) + out.append(serial_console.read()) + except Exception as e: + raise LisaException(f"Failed to execute commands: {e}") from e + finally: + serial_console.close() return out def _add_newline(self, command: str) -> str: From 83abbd5e57a5e8d35a410d67f72c67fe8ebb269c Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Tue, 15 Jul 2025 10:43:12 +0000 Subject: [PATCH 12/14] RunCommand to run one command at a time due to return size limitation --- lisa/features/non_ssh_executor.py | 2 +- lisa/node.py | 10 +++++---- lisa/sut_orchestrator/azure/features.py | 28 ++++++++++++++++++++----- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/lisa/features/non_ssh_executor.py b/lisa/features/non_ssh_executor.py index 0c413a6393..8d9d4bf71c 100644 --- a/lisa/features/non_ssh_executor.py +++ b/lisa/features/non_ssh_executor.py @@ -49,11 +49,11 @@ def _execute(self, commands: List[str]) -> List[str]: for command in commands: serial_console.write(self._add_newline(command)) out.append(serial_console.read()) + return out except Exception as e: raise LisaException(f"Failed to execute commands: {e}") from e finally: serial_console.close() - return out def _add_newline(self, command: str) -> str: """ diff --git a/lisa/node.py b/lisa/node.py index ac1bc5a547..36a34031e6 100644 --- a/lisa/node.py +++ b/lisa/node.py @@ -747,7 +747,7 @@ def check_sudo_password_required(self) -> None: raise RequireUserPasswordException("Reset password failed") self._check_password_and_store_prompt() - def _collect_logs_using_non_ssh_executor(self): + def _collect_logs_using_non_ssh_executor(self) -> None: """ Collects information using the NonSshExecutor feature. This is used when the connection to the node is not stable. @@ -768,9 +768,11 @@ def _collect_logs_using_non_ssh_executor(self): if self.features.is_supported(NonSshExecutor): non_ssh_executor = self.features[NonSshExecutor] - out = non_ssh_executor.execute(commands=commands) - out = "\n\n".join(out) - self.log.info(f"Collected information using NonSshExecutor:\n{out}") + outputs = non_ssh_executor.execute(commands=commands) + collected_info = "\n\n".join(outputs) + self.log.info( + f"Collected information using NonSshExecutor:\n{collected_info}" + ) else: self.log.debug( f"NonSshExecutor is not supported on {self.name}, " diff --git a/lisa/sut_orchestrator/azure/features.py b/lisa/sut_orchestrator/azure/features.py index 852106935f..629eb8357a 100644 --- a/lisa/sut_orchestrator/azure/features.py +++ b/lisa/sut_orchestrator/azure/features.py @@ -27,6 +27,7 @@ HardwareProfile, NetworkInterfaceReference, RunCommandInput, + RunCommandResult, VirtualMachineExtension, VirtualMachineUpdate, ) @@ -3767,7 +3768,7 @@ def execute(self, commands: List[str]) -> str: :return: The output of the commands. """ context = get_node_context(self._node) - platform = self._platform + platform: AzurePlatform = self._platform # type: ignore compute_client = get_compute_client(platform) # Prepare the RunCommandInput for Azure @@ -3783,8 +3784,21 @@ def execute(self, commands: List[str]) -> str: parameters=run_command_input, ) result = wait_operation(operation=operation, failure_identity="run command") + try: + # Since wait_operation returns a dict (result.as_dict()), access as dict + value = result.get("value") + if value and len(value) > 0 and value[0].get("message"): + message = value[0]["message"] + else: + raise LisaException( + "RunCommand did not run successfully. " + f"Got response: '{value}'. Expected response to contain `value[0]['message']`" + ) - return result["value"][0]["message"] + except Exception: + self._log.info("RunCommand failed to return expected result.") + raise + return message def _add_echo_before_command(self, commands: List[str]) -> List[str]: """ @@ -3796,10 +3810,14 @@ def _add_echo_before_command(self, commands: List[str]) -> List[str]: class NonSshExecutor(AzureFeatureMixin, features.NonSshExecutor): def execute(self, commands: List[str]) -> List[str]: - # RunCommand is faster than SerialConsole. Hence attempt to use it first. + # RunCommand does not require password login, hence attempt to use it first. + # RunCommand has a limitation on 4KB of output. try: - output = self._node.features[RunCommand].execute(commands) - return [output] + result = [] + for command in commands: + out = self._node.features[RunCommand].execute([command]) + result.append(out) + return result except Exception as e: self._log.info(f"RunCommand failed: {e}") # Fallback to the default non-SSH executor behavior From 737ca414f256224816d9783fffb3b0c96d09e9df Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Mon, 11 Aug 2025 06:02:03 +0000 Subject: [PATCH 13/14] Update review comments --- lisa/node.py | 6 +----- lisa/sut_orchestrator/azure/features.py | 21 ++++++++------------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/lisa/node.py b/lisa/node.py index 36a34031e6..344a47dda9 100644 --- a/lisa/node.py +++ b/lisa/node.py @@ -768,11 +768,7 @@ def _collect_logs_using_non_ssh_executor(self) -> None: if self.features.is_supported(NonSshExecutor): non_ssh_executor = self.features[NonSshExecutor] - outputs = non_ssh_executor.execute(commands=commands) - collected_info = "\n\n".join(outputs) - self.log.info( - f"Collected information using NonSshExecutor:\n{collected_info}" - ) + non_ssh_executor.execute(commands=commands) else: self.log.debug( f"NonSshExecutor is not supported on {self.name}, " diff --git a/lisa/sut_orchestrator/azure/features.py b/lisa/sut_orchestrator/azure/features.py index 629eb8357a..4b6e1bc8e2 100644 --- a/lisa/sut_orchestrator/azure/features.py +++ b/lisa/sut_orchestrator/azure/features.py @@ -3784,20 +3784,15 @@ def execute(self, commands: List[str]) -> str: parameters=run_command_input, ) result = wait_operation(operation=operation, failure_identity="run command") - try: - # Since wait_operation returns a dict (result.as_dict()), access as dict - value = result.get("value") - if value and len(value) > 0 and value[0].get("message"): - message = value[0]["message"] - else: - raise LisaException( - "RunCommand did not run successfully. " - f"Got response: '{value}'. Expected response to contain `value[0]['message']`" - ) + value = result.get("value") + if value and value[0].get("message"): + message = value[0]["message"] + else: + raise LisaException( + "RunCommand did not run successfully. " + f"Got response: '{value}'. Expected response to contain `value[0]['message']`" + ) - except Exception: - self._log.info("RunCommand failed to return expected result.") - raise return message def _add_echo_before_command(self, commands: List[str]) -> List[str]: From c8cd14633a373b0b7b7ab9f173ac297b211dfb34 Mon Sep 17 00:00:00 2001 From: Aditya Nagesh Date: Mon, 11 Aug 2025 06:28:40 +0000 Subject: [PATCH 14/14] Update Serial Console prompt check --- lisa/features/non_ssh_executor.py | 46 +++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/lisa/features/non_ssh_executor.py b/lisa/features/non_ssh_executor.py index 8d9d4bf71c..653667d483 100644 --- a/lisa/features/non_ssh_executor.py +++ b/lisa/features/non_ssh_executor.py @@ -1,3 +1,4 @@ +import re from typing import List from lisa.feature import Feature @@ -44,17 +45,58 @@ def _execute(self, commands: List[str]) -> List[str]: # write a newline and read to make sure serial console has the prompt serial_console.write("\n") response = serial_console.read() - if not response or "$" not in response and "#" not in response: - raise LisaException("Serial console prompt not found in output") + + # Check for full prompt pattern instead of individual characters + if not self._is_valid_prompt(response): + raise LisaException( + f"Valid shell prompt not found in output. " + f"Expected a shell prompt ending with $, #, or >, " + f"but got: {response.strip()}" + ) + for command in commands: serial_console.write(self._add_newline(command)) out.append(serial_console.read()) + collected_info = "\n\n".join(out) + self._log.info( + f"Collected information using NonSshExecutor:\n{collected_info}" + ) return out except Exception as e: raise LisaException(f"Failed to execute commands: {e}") from e finally: serial_console.close() + def _is_valid_prompt(self, response: str) -> bool: + """ + Check if the response contains a valid shell prompt pattern. + + :param response: The response from the serial console + :return: True if a valid prompt is found, False otherwise + """ + if not response: + return False + + # Generic pattern that matches any prompt format: + # - Username and hostname part: word chars, @, hyphens, dots + # - Colon separator + # - Path part: ~, /, word chars, dots, hyphens, slashes + # - Optional whitespace + # - Ending with $, #, or > + # - Optional trailing whitespace + prompt_pattern = r"[a-zA-Z0-9_@.-]+:[~/a-zA-Z0-9_./-]*\s*[\$#>]\s*$" + + # Check each line in the response for the prompt pattern + lines = response.split("\n") + for line in lines: + line = line.strip() + if re.search(prompt_pattern, line): + self._log.debug(f"Valid prompt found: '{line}'") + return True + + self._log.debug(f"No valid prompt found in response: '{response.strip()}'") + return False + def _add_newline(self, command: str) -> str: """ Adds a newline character to the command if it does not already end with one.