Skip to content

Commit 6836866

Browse files
dhsrivasAmandeep-Kaur2202
dhsrivas
authored and
Amandeep-Kaur2202
committed
Add guest boot sanity tests with passthrough device support
Add test and configurations to validate the following scenarios: 1. Launch guests in different guest interrupt modes: a. AVIC b. APIC c. x2APIC d. x2AVIC 2. Boot the above guests with vCPU counts: 512, 288, 256, 254, 128, 64 3. Boot guests with PCI passthrough devices. 4. Boot guests with emulated AMD IOMMU and emulated Intel IOMMU. Notes: 1. AVIC and APIC support fewer than 255 vCPUs. 2. x2APIC and x2AVIC modes require extended-apicid=on, or the presence of emulated AMD IOMMU / emulated Intel IOMMU, to support booting with >255 vCPUs. Signed-off-by: Dheeraj Kumar Srivastava <dheerajkumar.srivastava@amd.com>
1 parent 5997bad commit 6836866

File tree

5 files changed

+384
-0
lines changed

5 files changed

+384
-0
lines changed

‎qemu/tests/cfg/emulated_amd_iommu.cfg‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
- emulated_amd_iommu:
2+
type = qemu_pci_passthrough
3+
start_vm = no
4+
only Linux
5+
only x86_64
6+
only HostCpuVendor.amd
7+
pcie_extra_root_port = 0
8+
vt_ulimit_nofile = 65536
9+
usb_devices = ""
10+
auto_cpu_model = "no"
11+
cpu_model = "host"
12+
machine_type = "q35"
13+
machine_type_extra_params = "kernel-irqchip=split"
14+
extra_params = " -global kvm-pit.lost_tick_policy=discard"
15+
vcpu_sockets = 1
16+
vcpu_dies = 1
17+
vcpu_threads = 1
18+
mem = 8G
19+
mode = "x2apic"
20+
variants:
21+
- no_passthrough:
22+
required_qemu = [8.2.90,)
23+
# TODO Add passthrough variant
24+
variants:
25+
- vcpu_512:
26+
smp_fixed = 512
27+
vcpu_maxcpus = 512
28+
vcpu_cores = 512
29+
- vcpu_288:
30+
smp_fixed = 288
31+
vcpu_maxcpus = 288
32+
vcpu_cores = 288
33+
- vcpu_256:
34+
smp_fixed = 256
35+
vcpu_maxcpus = 256
36+
vcpu_cores = 256
37+
- vcpu_254:
38+
smp_fixed = 254
39+
vcpu_maxcpus = 254
40+
vcpu_cores = 254
41+
- vcpu_128:
42+
smp_fixed = 128
43+
vcpu_maxcpus = 128
44+
vcpu_cores = 128
45+
- vcpu_64:
46+
smp_fixed = 64
47+
vcpu_maxcpus = 64
48+
vcpu_cores = 64
49+
variants:
50+
- x2apic_emul_amd_iommu:
51+
cpu_model_flags += ",+x2apic"
52+
extra_params += " -device amd-iommu,intremap=on,xtsup=on"
53+
kvm_probe_module_parameters = "avic=0"
54+
- x2avic_emul_amd_iommu:
55+
cpu_model_flags += ",+x2apic"
56+
extra_params += " -device amd-iommu,intremap=on,xtsup=on"
57+
kvm_probe_module_parameters = "avic=1"

‎qemu/tests/cfg/iommu_accelerated_guest_mode.cfg‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
- iommu_accelerated_guest_mode:
2+
type = qemu_pci_passthrough
3+
start_vm = no
4+
only Linux
5+
only x86_64
6+
only HostCpuVendor.amd
7+
pcie_extra_root_port = 0
8+
vt_ulimit_nofile = 65536
9+
usb_devices = ""
10+
auto_cpu_model = "no"
11+
cpu_model = "host"
12+
machine_type = "q35"
13+
machine_type_extra_params = "kernel-irqchip=split"
14+
extra_params = " -global kvm-pit.lost_tick_policy=discard"
15+
vcpu_sockets = 1
16+
vcpu_dies = 1
17+
vcpu_threads = 1
18+
mem = 8G
19+
mode = "x2apic"
20+
kvm_probe_module_parameters = "avic=1"
21+
# Update pci_device with host PCI device/s for passthrough to the guest
22+
# enabling IOMMU functionality validation. eg. pci_device = "0000:01:00.0"
23+
# Note: Without a PCI device input, the guest will still boot, but the
24+
# test won't exercise the host IOMMU.
25+
#pci_device = ""
26+
variants:
27+
- vcpu_512:
28+
smp_fixed = 512
29+
vcpu_maxcpus = 512
30+
vcpu_cores = 512
31+
- vcpu_288:
32+
smp_fixed = 288
33+
vcpu_maxcpus = 288
34+
vcpu_cores = 288
35+
- vcpu_256:
36+
smp_fixed = 256
37+
vcpu_maxcpus = 256
38+
vcpu_cores = 256
39+
- vcpu_254:
40+
smp_fixed = 254
41+
vcpu_maxcpus = 254
42+
vcpu_cores = 254
43+
- vcpu_128:
44+
smp_fixed = 128
45+
vcpu_maxcpus = 128
46+
vcpu_cores = 128
47+
- vcpu_64:
48+
smp_fixed = 64
49+
vcpu_maxcpus = 64
50+
vcpu_cores = 64
51+
variants:
52+
- avic:
53+
mode = "apic"
54+
cpu_model_flags += ",-x2apic"
55+
no vcpu_512, vcpu_288, vcpu_256
56+
- x2avic_ext_apic_id:
57+
cpu_model_flags += ",+x2apic"
58+
cpu_model_flags += ",kvm-msi-ext-dest-id=on"
59+
- x2avic_emul_intel_iommu:
60+
intel_iommu = yes
61+
iommu_eim = on
62+
iommu_intremap = on
63+
guest_iommu_option = pt
64+
iommu_caching_mode = on
65+
cpu_model_flags += ",+x2apic"

‎qemu/tests/cfg/iommu_guest_mode.cfg‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
- iommu_guest_mode:
2+
type = qemu_pci_passthrough
3+
start_vm = no
4+
only Linux
5+
only x86_64
6+
only HostCpuVendor.amd
7+
pcie_extra_root_port = 0
8+
vt_ulimit_nofile = 65536
9+
usb_devices = ""
10+
auto_cpu_model = "no"
11+
cpu_model = "host"
12+
machine_type = "q35"
13+
machine_type_extra_params = "kernel-irqchip=split"
14+
extra_params = ""
15+
vcpu_sockets = 1
16+
vcpu_dies = 1
17+
vcpu_threads = 1
18+
mem = 8G
19+
mode = "x2apic"
20+
kvm_probe_module_parameters = "avic=0"
21+
# Update pci_device with host PCI device/s for passthrough to the guest
22+
# enabling IOMMU functionality validation. eg. pci_device = "0000:01:00.0"
23+
# Note: Without a PCI device input, the guest will still boot, but the
24+
# test won't exercise the host IOMMU.
25+
#pci_device = ""
26+
variants:
27+
- vcpu_1024:
28+
smp_fixed = 1024
29+
vcpu_maxcpus = 1024
30+
vcpu_cores = 1024
31+
- vcpu_512:
32+
smp_fixed = 512
33+
vcpu_maxcpus = 512
34+
vcpu_cores = 512
35+
- vcpu_288:
36+
smp_fixed = 288
37+
vcpu_maxcpus = 288
38+
vcpu_cores = 288
39+
- vcpu_256:
40+
smp_fixed = 256
41+
vcpu_maxcpus = 256
42+
vcpu_cores = 256
43+
- vcpu_254:
44+
smp_fixed = 254
45+
vcpu_maxcpus = 254
46+
vcpu_cores = 254
47+
- vcpu_128:
48+
smp_fixed = 128
49+
vcpu_maxcpus = 128
50+
vcpu_cores = 128
51+
- vcpu_64:
52+
smp_fixed = 64
53+
vcpu_maxcpus = 64
54+
vcpu_cores = 64
55+
variants:
56+
- apic:
57+
mode = "apic"
58+
cpu_model_flags += ",-x2apic"
59+
no vcpu_1024, vcpu_512, vcpu_288, vcpu_256
60+
- x2apic_ext_apic_id:
61+
cpu_model_flags += ",+x2apic"
62+
cpu_model_flags += ",kvm-msi-ext-dest-id=on"
63+
- x2apic_emul_intel_iommu:
64+
intel_iommu = yes
65+
iommu_eim = on
66+
iommu_intremap = on
67+
guest_iommu_option = pt
68+
iommu_caching_mode = on
69+
cpu_model_flags += ",+x2apic"

‎qemu/tests/cfg/vfio_pci_passthrough.cfg‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
- vfio_pci_passthrough:
2+
type = qemu_pci_passthrough
3+
start_vm = no
4+
only Linux
5+
only x86_64
6+
only HostCpuVendor.amd
7+
pcie_extra_root_port = 0
8+
vt_ulimit_nofile = 65536
9+
usb_devices = ""
10+
auto_cpu_model = "no"
11+
cpu_model = "host"
12+
machine_type = "q35"
13+
extra_params = ""
14+
mem = 8G
15+
# Update pci_device with host PCI device/s for passthrough to the guest
16+
# enabling IOMMU functionality validation. eg. pci_device = "0000:01:00.0"
17+
# Note: Without a PCI device input, the guest will still boot, but the
18+
# test won't exercise the host IOMMU.
19+
#pci_device = ""

‎qemu/tests/qemu_pci_passthrough.py‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
"""
2+
Guest boot sanity test with passthrough device in different mode
3+
- Apic, X2apic, Avic, X2avic
4+
"""
5+
6+
from avocado.utils import dmesg, linux_modules, pci, process
7+
from virttest import env_process
8+
9+
10+
def run(test, params, env): # pylint: disable=R0915
11+
"""
12+
Guest boot sanity test with passthrough device
13+
Steps:
14+
1. Launch a guest with a vfio pci passthrough device
15+
2. Verify if vfio pci passthrough of device to guest is successful
16+
and guest boots in expected mode - apic, avic, x2apic and x2avic.
17+
18+
:param test: QEMU test object.
19+
:param params: Dictionary with test parameters:
20+
- pci_device: Pci device to passthrough. Default: ""
21+
- mode: Defines guest modes - [apic, x2apic]. Default: x2apic.
22+
- kvm_probe_module_parameters: To enable/disable avic on host.
23+
Possible values ["avic=1", "avic=0"]
24+
- login_timeout: VM login timeout in seconds. Default: 240).
25+
:param env: Dictionary with test environment.
26+
:raises: cancel if
27+
1. At start of test, KVM module is not loaded.
28+
2. KVM, msr, vfio_pci is not built or loadable.
29+
3. pci_device != ""
30+
a. vfio_pci is not built or loadable.
31+
b. Interrupt remapping is not enabled on host. Required for
32+
passthrough.
33+
c. Pci device/s not found on host system.
34+
4. kvm_probe_module_parameters = "avic=1"
35+
a. Host doesnot support expected mode - avic or x2avic.
36+
b. Host doesnot have expected mode enabled for guest - avic
37+
or x2avic.
38+
fails if
39+
1. Pci device cannot be bind to vfio_pci module for passthrough.
40+
(pci_device != "")
41+
2. Unable to login to guest within login timeout.
42+
"""
43+
kvm_probe_module_parameters = params.get("kvm_probe_module_parameters", "")
44+
login_timeout = int(params.get("login_timeout", 240))
45+
pci_device = params.get("pci_device", "")
46+
mode = params.get("mode", "x2apic")
47+
driver_list = []
48+
session = None
49+
vm = None
50+
51+
try:
52+
53+
def check_avic_support():
54+
"""
55+
Check if system supports avic.
56+
"""
57+
cmd = "rdmsr -p 0 0xc00110dd --bitfield 13:13"
58+
out = process.run(cmd, sudo=True, shell=True).stdout_text.strip()
59+
if out == "0":
60+
test.cancel("System doesnot support avic")
61+
62+
def check_x2avic_support():
63+
"""
64+
check if system supports x2avic.
65+
"""
66+
cmd = "rdmsr -p 0 0xc00110dd --bitfield 18:18"
67+
out = process.run(cmd, sudo=True, shell=True).stdout_text.strip()
68+
if out == "0":
69+
test.cancel("System doesnot support x2avic")
70+
71+
def verify_avic_enablement(mode):
72+
"""
73+
Check AVIC and x2AVIC status in dmesg logs diff.
74+
75+
:param mode: Whether apic or x2apic.
76+
"""
77+
78+
# Check for the "avic enabled" in dmesg
79+
if not dmesg.check_kernel_logs("AVIC enabled"):
80+
test.cancel("AVIC not enabled after loading kvm_amd with avic=1")
81+
82+
# Check for the "x2avic enabled" only if the test is "x2apic"
83+
if (not dmesg.check_kernel_logs("x2AVIC enabled")) and (mode == "x2apic"):
84+
test.cancel("x2AVIC not enabled after loading kvm_amd with avic=1.")
85+
86+
def prepare_pci_passthrough():
87+
"""
88+
Validate IOMMU, Interrupt Remapping, and vfio-pci module availability
89+
"""
90+
91+
# Check if interrupt remapping is enabled on system
92+
if not dmesg.check_kernel_logs("AMD-Vi: Interrupt remapping enabled"):
93+
test.cancel("IOMMU interrupt remaping is not enabled")
94+
95+
# Check and load vfio-pci module
96+
linux_modules.configure_module("vfio-pci", "CONFIG_VFIO_PCI")
97+
98+
def guest_system_details(session):
99+
"""
100+
Collect guest system details
101+
102+
:param session: active guest login session
103+
"""
104+
test.log.debug("Debug: %s", session.cmd_output("cat /etc/os-release"))
105+
test.log.debug("Debug: %s", session.cmd_output("uname -a"))
106+
test.log.debug("Debug: %s", session.cmd_output("ls /boot/"))
107+
test.log.debug("Debug: %s", session.cmd_output("lspci -k"))
108+
test.log.debug("Debug: %s", session.cmd_output("lscpu"))
109+
test.log.debug("Debug: %s", session.cmd_output("lsblk"))
110+
test.log.debug("Debug: %s", session.cmd_output("df -h"))
111+
test.log.debug("Debug: %s", session.cmd_output("dmesg"))
112+
113+
# Check system support for avic or x2avic
114+
if kvm_probe_module_parameters == "avic=1":
115+
linux_modules.configure_module("msr", "CONFIG_X86_MSR")
116+
if mode == "apic":
117+
check_avic_support()
118+
if mode == "x2apic":
119+
check_x2avic_support()
120+
# Validate dmesg for avic and x2avic enablement
121+
verify_avic_enablement(mode)
122+
123+
# Passthrough device/s and validate if passthrough is successful
124+
if pci_device != "":
125+
# Perform pre-checks and prereq enablements before pci passthrough
126+
prepare_pci_passthrough()
127+
128+
# Prepare for pci passthrough
129+
for i in range(len(pci_device.split(" "))):
130+
# Check if device input is valid
131+
if pci_device.split(" ")[i] not in pci.get_pci_addresses():
132+
test.cancel("Please provide valid pci device input.")
133+
134+
driver_list.append(pci.get_driver(pci_device.split(" ")[i]))
135+
pci.attach_driver(pci_device.split(" ")[i], "vfio-pci")
136+
params["extra_params"] += (
137+
f" -device vfio-pci,host={pci_device.split(' ')[i]}"
138+
)
139+
140+
params["start_vm"] = "yes"
141+
vm = env.get_vm(params["main_vm"])
142+
try:
143+
env_process.preprocess_vm(test, params, env, params.get("main_vm"))
144+
vm.verify_alive()
145+
except Exception as e:
146+
test.fail(f"Failed to create VM: {str(e)}")
147+
try:
148+
session = vm.wait_for_login(timeout=login_timeout)
149+
except Exception as e:
150+
test.fail(f"Failed to login VM: {str(e)}")
151+
vm.verify_kernel_crash()
152+
153+
# Collect guest system details
154+
guest_system_details(session)
155+
except ValueError as e:
156+
test.fail(f"{e}")
157+
finally:
158+
if session:
159+
session.close()
160+
if vm:
161+
vm.destroy()
162+
try:
163+
if pci_device != "":
164+
for i in range(len(pci_device.split(" "))):
165+
if pci_device.split(" ")[i] not in pci.get_pci_addresses():
166+
break
167+
if driver_list[i] is None:
168+
cur_driver = pci.get_driver(pci_device.split(" ")[i])
169+
if cur_driver is not None:
170+
pci.unbind(cur_driver, pci_device.split(" ")[i])
171+
else:
172+
pci.attach_driver(pci_device.split(" ")[i], driver_list[i])
173+
except ValueError as e:
174+
test.fail(f"Failed to reset devices after test: Reason {e}")

0 commit comments

Comments
 (0)