From bd630c097ef168900856aecd87aab270e2be8c31 Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 8 Oct 2025 18:28:31 -0700 Subject: [PATCH 1/5] meson: allow update_envs and meson -D defines --- lisa/tools/meson.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lisa/tools/meson.py b/lisa/tools/meson.py index bcc6fb2782..d27d5f28b0 100644 --- a/lisa/tools/meson.py +++ b/lisa/tools/meson.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. from pathlib import PurePath -from typing import cast +from typing import List, Optional, cast from lisa.executable import Tool from lisa.operating_system import Posix @@ -97,9 +97,18 @@ def _install(self) -> bool: return self._check_exists() - def setup(self, args: str, cwd: PurePath, build_dir: str = "build") -> PurePath: + def setup( + self, + args: str, + cwd: PurePath, + build_dir: str = "build", + variables: Optional[List[str]] = None, + ) -> PurePath: + variable_defs = "" + if variables: + variable_defs = " ".join([f"-D{x}" for x in variables]) self.run( - f"{args} {build_dir}", + f"{args} {build_dir} {variable_defs}", force_run=True, shell=True, cwd=cwd, From 6ebee1ba9444fe0675686c6e5bc0d7fb8c108cfd Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 8 Oct 2025 18:28:41 -0700 Subject: [PATCH 2/5] timeout: allow update envs --- lisa/tools/timeout.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lisa/tools/timeout.py b/lisa/tools/timeout.py index 67ccc88c76..c429784fd6 100644 --- a/lisa/tools/timeout.py +++ b/lisa/tools/timeout.py @@ -1,3 +1,5 @@ +from typing import Dict, Optional + from lisa.executable import ExecutableResult, Process, Tool from lisa.util.constants import SIGTERM @@ -46,9 +48,16 @@ def start_with_timeout( signal: int = SIGTERM, kill_timeout: int = 0, delay_start: int = 0, + update_envs: Optional[Dict[str, str]] = None, ) -> Process: # timeout [OPTION] DURATION COMMAND [ARG]... params = f"-s {signal} --preserve-status {timeout} {command}" if kill_timeout: params = f"--kill-after {kill_timeout} " + params - return self.run_async(parameters=params, force_run=True, shell=True, sudo=True) + return self.run_async( + parameters=params, + force_run=True, + shell=True, + sudo=True, + update_envs=update_envs, + ) From 5d6aebc6a179cfa838429dc79b53bd51abc48161 Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 8 Oct 2025 18:30:13 -0700 Subject: [PATCH 3/5] dpdk: enable ASAN builds (unfinished) Implementing ASAN builds and invocations of DPDK, this requires: - LD_PRELOAD to force loading of the libasan.so library before others to hook malloc and free etc. - setting a meson build option -Db_sanitize=address - enlightening the Installer class to be aware of whether ASAN is desired. - passing a runbook option through to the DpdkTestpmd class from initialize_node_resources - setting ASAN_OPTIONS=detect_leaks=0 since we don't care about memory leaks during teardown. ASAN and DPDK are not enabled globally for all testing (yet). It's useful for debugging segfaults and memory corruption during patch testing so I'd like to make using it easier. testpmd --- microsoft/testsuites/dpdk/common.py | 12 +++++++++- microsoft/testsuites/dpdk/dpdktestpmd.py | 30 +++++++++++++++++++++--- microsoft/testsuites/dpdk/dpdkutil.py | 29 ++++++++++++++++++++++- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/microsoft/testsuites/dpdk/common.py b/microsoft/testsuites/dpdk/common.py index a009007262..1b4c0d3c4b 100644 --- a/microsoft/testsuites/dpdk/common.py +++ b/microsoft/testsuites/dpdk/common.py @@ -226,7 +226,9 @@ def _should_install(self, required_version: Optional[VersionInfo] = None) -> boo ) # run the defined setup and installation steps. - def do_installation(self, required_version: Optional[VersionInfo] = None) -> None: + def do_installation( + self, required_version: Optional[VersionInfo] = None, **kwargs: Any + ) -> None: self._setup_node() if self._should_install(): self._uninstall() @@ -238,6 +240,7 @@ def __init__( node: Node, os_dependencies: Optional[DependencyInstaller] = None, downloader: Optional[Downloader] = None, + **kwargs: Any, ) -> None: self._node = node if not isinstance(self._node.os, Posix): @@ -248,6 +251,9 @@ def __init__( self._package_manager_extra_args: List[str] = [] self._os_dependencies = os_dependencies self._downloader = downloader + self._kwargs = kwargs + # default to no asan, it's applicable for source builds only + self.use_asan = bool(kwargs.pop("use_asan", False)) # Base class for package manager installation @@ -379,6 +385,10 @@ def is_url_for_tarball(url: str) -> bool: return ".tar" in suffixes +def find_libasan_so(node: Node) -> str: + return node.execute("find /usr/lib/ -name libasan.so", sudo=True, shell=True).stdout + + def is_url_for_git_repo(url: str) -> bool: parsed_url = parse_url(url) scheme = parsed_url.scheme diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index 5583b847bd..c74b0c64b5 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -44,6 +44,7 @@ OsPackageDependencies, PackageManagerInstall, TarDownloader, + find_libasan_so, get_debian_backport_repo_args, is_url_for_git_repo, is_url_for_tarball, @@ -289,8 +290,16 @@ def _install(self) -> None: node = self._node # save the pythonpath for later python_path = node.tools[Python].get_python_path() + # pick meson options, add ASAN build arg if present. + meson_options = ["buildtype=debug"] + if self.use_asan: + meson_options += ["b_sanitize=address"] + # invoke meson self.dpdk_build_path = node.tools[Meson].setup( - args=sample_apps, build_dir="build", cwd=self.asset_path + args=sample_apps, + build_dir="build", + cwd=self.asset_path, + variables=meson_options, ) install_result = node.tools[Ninja].run( cwd=self.dpdk_build_path, @@ -735,7 +744,19 @@ def get_example_app_path(self, app_name: str) -> PurePath: # check if there is a build directory and build the application # (if necessary) if not shell.exists(source_path.joinpath("build")): - self.node.tools[Make].make("static", cwd=source_path, sudo=True) + libasan_so = find_libasan_so(self.node) + assert libasan_so != "", "couldn't find libasan" + envs = ( + { + "CFLAGS": "-fsanitize=address", + "ASAN_OPTIONS": "detect_leaks=false", + "LD_PRELOAD": libasan_so, + } + if self.installer.use_asan + else None + ) + self.node.tools[Make].make("", cwd=source_path, sudo=True, update_envs=envs) + return source_path.joinpath(f"build/{source_path.name}") def __init__(self, *args: Any, **kwargs: Any) -> None: @@ -749,7 +770,8 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self._determine_network_hardware() if self.use_package_manager_install(): self.installer: Installer = DpdkPackageManagerInstall( - self.node, DPDK_PACKAGE_MANAGER_PACKAGES + self.node, + DPDK_PACKAGE_MANAGER_PACKAGES, ) # if not package manager, choose source installation else: @@ -777,10 +799,12 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: " Expected https://___/___.git or /path/to/tar.tar[.gz] or " "https://__/__.tar[.gz]" ) + self._use_asan = bool(kwargs.pop("use_asan", False)) self.installer = DpdkSourceInstall( node=self.node, os_dependencies=DPDK_SOURCE_INSTALL_PACKAGES, downloader=downloader, + use_asan=self._use_asan, ) # if dpdk is already installed, find the binary and check the version if self.find_testpmd_binary(assert_on_fail=False): diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py index 4439946a82..790d9ab259 100644 --- a/microsoft/testsuites/dpdk/dpdkutil.py +++ b/microsoft/testsuites/dpdk/dpdkutil.py @@ -60,6 +60,7 @@ PackageManagerInstall, TarDownloader, check_dpdk_support, + find_libasan_so, is_url_for_git_repo, is_url_for_tarball, update_kernel_from_repo, @@ -267,6 +268,7 @@ def enable_uio_hv_generic(node: Node) -> None: uname = node.tools[Uname] # check if kernel config for Hyper-V VMBus is enabled + config = "CONFIG_UIO_HV_GENERIC" if not kconfig.is_enabled(config): kversion = uname.get_linux_information().kernel_version @@ -330,6 +332,7 @@ def initialize_node_resources( dpdk_branch = variables.get("dpdk_branch", "") rdma_source = variables.get("rdma_source", "") rdma_branch = variables.get("rdma_branch", "") + dpdk_use_asan = variables.get("dpdk_use_asan", False) force_net_failsafe_pmd = variables.get("dpdk_force_net_failsafe_pmd", False) log.info( "Dpdk initialize_node_resources running" @@ -372,6 +375,7 @@ def initialize_node_resources( dpdk_branch=dpdk_branch, sample_apps=sample_apps, force_net_failsafe_pmd=force_net_failsafe_pmd, + use_asan=dpdk_use_asan, ) # Tools will skip installation if the binary is present, so # force invoke install. Installer will skip if the correct @@ -1344,6 +1348,7 @@ class DpdkDevnameInfo: def __init__(self, testpmd: DpdkTestpmd) -> None: self._node = testpmd.node self._testpmd = testpmd + self.env_args = None def get_port_info(self, nics: List[NicInfo], expect_ports: int = 1) -> str: # since we only need this for netvsc, we'll only generate @@ -1351,6 +1356,15 @@ def get_port_info(self, nics: List[NicInfo], expect_ports: int = 1) -> str: # This is needed because the port_ids will change # depending on how many NICs are present _and_ enabled # by the EAL. + self.env_args = ( + { + "ASAN_OPTIONS": "detect_leaks=false", + "LD_PRELOAD": f"{find_libasan_so(self._node)}", + } + if self._testpmd.installer.use_asan + else None + ) + if self._node.nics.is_mana_device_present(): # mana needs a vdev argument of pci info # followed by kv pairs for mac addresses. @@ -1370,9 +1384,10 @@ def get_port_info(self, nics: List[NicInfo], expect_ports: int = 1) -> str: # run the application with the device include arguments. output = self._node.execute( - f"{str(self._testpmd.get_example_app_path('devname'))} {nic_args}", + (f"{str(self._testpmd.get_example_app_path('devname'))} {nic_args}"), sudo=True, shell=True, + update_envs=self.env_args, ).stdout # find all the matches for devices bound to net_netvsc PMD @@ -1520,6 +1535,16 @@ def run_dpdk_symmetric_mp( "--log-level vmbus,debug " f"-- -p {port_mask} --num-procs 2" ) + libasan_so = find_libasan_so(test_kit.node) + assert libasan_so != "", "Test bug: libasan.so is missing after source build." + symmetric_mp_envs = ( + { + "LD_PRELOAD": libasan_so, + "ASAN_OPTIONS": "detect_leaks=false ", + } + if test_kit.testpmd.installer.use_asan + else None + ) # start the first process (id 0) on core 1 primary = node.tools[Timeout].start_with_timeout( command=( @@ -1529,6 +1554,7 @@ def run_dpdk_symmetric_mp( timeout=660, signal=SIGINT, kill_timeout=30, + update_envs=symmetric_mp_envs, ) # wait for it to start @@ -1543,6 +1569,7 @@ def run_dpdk_symmetric_mp( timeout=600, signal=SIGINT, kill_timeout=35, + update_envs=symmetric_mp_envs, ) secondary.wait_output("APP: Finished Process Init", timeout=20) From 562bcad2a57fea4d4514f77da43127e0c8fe68a3 Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 15 Oct 2025 09:52:39 -0700 Subject: [PATCH 4/5] timeout: allow update envs --- lisa/tools/timeout.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lisa/tools/timeout.py b/lisa/tools/timeout.py index c429784fd6..b05748ec01 100644 --- a/lisa/tools/timeout.py +++ b/lisa/tools/timeout.py @@ -21,6 +21,7 @@ def run_with_timeout( timeout: int, signal: int = SIGTERM, kill_timeout: int = 0, + update_envs: Optional[Dict[str, str]] = None, ) -> ExecutableResult: # timeout [OPTION] DURATION COMMAND [ARG]... @@ -39,6 +40,7 @@ def run_with_timeout( timeout=timeout, signal=signal, kill_timeout=kill_timeout, + update_envs=update_envs, ).wait_result(timeout=command_timeout) def start_with_timeout( From 9953e93cba5a705e139b24c1742e2ba03745fc9e Mon Sep 17 00:00:00 2001 From: "Matthew McGovern (LINUX)" Date: Wed, 15 Oct 2025 09:53:01 -0700 Subject: [PATCH 5/5] dpdk: set env args for testpmd asan --- microsoft/testsuites/dpdk/dpdktestpmd.py | 18 +++++++++++------- microsoft/testsuites/dpdk/dpdkutil.py | 3 ++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/microsoft/testsuites/dpdk/dpdktestpmd.py b/microsoft/testsuites/dpdk/dpdktestpmd.py index c74b0c64b5..38aa230a99 100644 --- a/microsoft/testsuites/dpdk/dpdktestpmd.py +++ b/microsoft/testsuites/dpdk/dpdktestpmd.py @@ -3,7 +3,7 @@ import re from pathlib import PurePath, PurePosixPath -from typing import Any, List, Tuple, Type +from typing import Any, Dict, List, Optional, Tuple, Type from assertpy import assert_that, fail from semver import VersionInfo @@ -615,9 +615,16 @@ def generate_testpmd_command( def run_for_n_seconds(self, cmd: str, timeout: int) -> str: self._last_run_timeout = timeout self.node.log.info(f"{self.node.name} running: {cmd}") - + envs: Optional[Dict[str, str]] = ( + { + "ASAN_OPTIONS": "detect_leaks=false", + "LD_PRELOAD": find_libasan_so(self.node), + } + if self.installer.use_asan + else None + ) proc_result = self.node.tools[Timeout].run_with_timeout( - cmd, timeout, SIGINT, kill_timeout=timeout + 10 + cmd, timeout, SIGINT, kill_timeout=timeout + 10, update_envs=envs ) self._last_run_output = proc_result.stdout self.populate_performance_data() @@ -626,7 +633,6 @@ def run_for_n_seconds(self, cmd: str, timeout: int) -> str: def start_for_n_seconds(self, cmd: str, timeout: int) -> str: self._last_run_timeout = timeout self.node.log.info(f"{self.node.name} running: {cmd}") - proc_result = self.node.tools[Timeout].run_with_timeout( cmd, timeout, SIGINT, kill_timeout=timeout + 10 ) @@ -746,11 +752,9 @@ def get_example_app_path(self, app_name: str) -> PurePath: if not shell.exists(source_path.joinpath("build")): libasan_so = find_libasan_so(self.node) assert libasan_so != "", "couldn't find libasan" - envs = ( + envs: Optional[Dict[str, str]] = ( { "CFLAGS": "-fsanitize=address", - "ASAN_OPTIONS": "detect_leaks=false", - "LD_PRELOAD": libasan_so, } if self.installer.use_asan else None diff --git a/microsoft/testsuites/dpdk/dpdkutil.py b/microsoft/testsuites/dpdk/dpdkutil.py index 790d9ab259..7b33f9d8f7 100644 --- a/microsoft/testsuites/dpdk/dpdkutil.py +++ b/microsoft/testsuites/dpdk/dpdkutil.py @@ -496,6 +496,7 @@ def _run_command_with_testkit( run_kit: Tuple[DpdkTestResources, str], ) -> Tuple[DpdkTestResources, str]: testkit, cmd = run_kit + return (testkit, testkit.testpmd.run_for_n_seconds(cmd, seconds)) task_manager = run_in_parallel_async( @@ -1348,7 +1349,7 @@ class DpdkDevnameInfo: def __init__(self, testpmd: DpdkTestpmd) -> None: self._node = testpmd.node self._testpmd = testpmd - self.env_args = None + self.env_args: Optional[Dict[str, str]] = None def get_port_info(self, nics: List[NicInfo], expect_ports: int = 1) -> str: # since we only need this for netvsc, we'll only generate