Adapted to pytest framework

Author: Menglths

test/common/llmperf/run_inference.py

@@ -128,7 +128,7 @@ def inference_results():
         server_url = config.get("llm_connection", {}).get("server_url", "")
         tokenizer_path = config.get("llm_connection", {}).get("tokenizer_path", "")
         test_cases = config.get("llmperf_test_cases", [])
-        timestamp_dir = Path("result_outputs")
+        timestamp_dir = Path("results")
         timestamp_dir.mkdir(parents=True, exist_ok=True)
         print(f"[INFO] Created results directory: {timestamp_dir}")
 

test/common/llmperf/utils/token_benchmark.py

@@ -353,8 +353,14 @@ def run_token_benchmark(
         elif not results_dir.is_dir():
             raise ValueError(f"{results_dir} is not a directory")
 
+        llmperf_dir = results_dir / "llmperf"
+        if not llmperf_dir.exists():
+            llmperf_dir.mkdir(parents=True)
+        elif not llmperf_dir.is_dir():
+            raise ValueError(f"{llmperf_dir} is not a directory")
+
         try:
-            with open(results_dir / f"{summary_filename}.json", "w") as f:
+            with open(llmperf_dir / f"{summary_filename}.json", "w") as f:
                 json.dump(results.to_dict(), f, indent=4, default=str)
         except Exception as e:
             print(results.to_dict())

test/config.yaml

@@ -15,4 +15,19 @@ database:
   name: "ucm_pytest"
   user: "root"
   password: "123456"
-  charset: "utf8mb4"
+  charset: "utf8mb4"
+
+# LLM Connection Configuration
+llm_connection:
+  model: "qwen3"
+  server_url: "http://141.111.32.70:9382"
+  tokenizer_path: "/home/models/QwQ-32B"
+
+# Performance Test Configuration
+llmperf_test_cases:
+  - mean_input_tokens: 6000
+    mean_output_tokens: 200
+    max_num_completed_requests: 16
+    concurrent_requests: 8
+    additional_sampling_params: "{}"
+    hit_rate: 0

test/suites/E2E/test_demo_function.py

@@ -1,185 +1,66 @@
-# tests/test_demo.py
 import pytest
-import allure
+from common.config_utils import config_utils as config_instance
 
-@pytest.mark.stage(1)
-@pytest.mark.feature("mark")
-@pytest.mark.platform("gpu")
-def test_gpu_smoke():
-    assert 1 == 1
 
-@pytest.mark.stage(1)
-@pytest.mark.feature("mark")
-def test_regress_accuracy():
-    assert 2 + 2 <= 5
+# ---------------- Fixture Example ----------------
+class Calculator:
+    def __init__(self):
+        print("[Calculator Initialization]")
+        pass
+
+    def add(self, a, b):
+        return a + b
+
+    def divide(self, a, b):
+        if b == 0:
+            raise ZeroDivisionError("Cannot divide by zero")
+        return a / b
+
+
+@pytest.fixture(scope="module", name="calc")
+def calculator():
+    return Calculator()
 
-@pytest.mark.stage(1)
-@pytest.mark.feature("mark")
-@pytest.mark.platform("npu")
-def test_performance_accuracy():
-    assert 2 + 2 <= 5
 
-# Example of new mark
 @pytest.mark.feature("mark")
-@pytest.mark.reliability("high")
-def test_llm_reliability():
-    assert True
+class TestCalculator:
+    # The calc instance will only be initialized on the first call, see the pytest documentation for more usage
+    def test_add(self, calc):
+        assert calc.add(1, 2) == 3
 
+    def test_divide(self, calc):
+        assert calc.divide(6, 2) == 3
 
-# Example of importing configuration file parameters
+    def test_divide_by_zero(self, calc):
+        with pytest.raises(ZeroDivisionError):
+            calc.divide(6, 0)
+
+
+# ---------------- Write to DB Example ----------------
+from common.capture_utils import *
+
+
+@pytest.mark.feature("capture")  # pytest must be the top
+@export_vars
+def test_capture_mix():
+    """Mixed single + lists via '_name' + '_data'"""
+    assert 1 == 1
+    return {
+        "_name": "demo",
+        "_data": {
+            "length": 10086,  # single value
+            "accuracy": [0.1, 0.2, 0.3],  # list
+            "loss": [0.1, 0.2, 0.3],  # list
+        },
+    }
+
+
+# ---------------- Read Config Example ----------------
 from common.config_utils import config_utils as config_instance
+
+
 @pytest.mark.feature("config")
-def test_llm_config():  
-    llm_config = config_instance.get_config("llm_connection")
-    assert llm_config["type"] == "openai"
-    assert config_instance.get_nested_config("llm_connection.model") == "gpt-3.5-turbo"
-    assert config_instance.get_nested_config("llm_connection.models", "gpt-3.5-turbo") == "gpt-3.5-turbo"
-
-
-
-# Example of using allure
-@pytest.mark.feature("allure1")
-@allure.feature('test_success')
-def test_success():
-    """this test succeeds"""
-    assert True
-
-@allure.feature('test_failure')
-@pytest.mark.feature("allure1")
-def test_failure():
-    """this test fails"""
-    assert False
-
-@allure.feature('test_skip')
-@pytest.mark.feature("allure1")
-def test_skip():
-    """this test is skipped"""
-    pytest.skip('for a reason!')
-
-@allure.feature('test_broken')
-@pytest.mark.feature("allure1")
-def test_broken():
-    raise Exception('oops')
-
-@pytest.mark.feature("allure2")
-@pytest.mark.parametrize('param1', ["Hello", "World"])
-@pytest.mark.parametrize('param2', ['Hello', "Hello"])
-def test_parametrize_with_two_parameters(param1, param2):
-    assert param1 == param2
-
-@pytest.mark.feature("allure3")
-@allure.description_html("""
-<h1>This is HTML description</h1>
-<table style="width:100%">
-  <tr>
-    <th>Firstname</th>
-    <th>Lastname</th>
-    <th>Age</th>
-  </tr>
-  <tr align="center">
-    <td>jade</td>
-    <td>mr</td>
-    <td>18</td>
-  </tr>
-  <tr align="center">
-    <td>road</td>
-    <td>Tester</td>
-    <td>18</td>
-  </tr>
-</table>
-""")
-def test_html_description():
-    assert True
-
-@pytest.mark.feature("allure3")
-@allure.description("""Multi-line description""")
-def test_description_from_decorator():
-    assert 42 == int(6 * 7)
-
-@pytest.mark.feature("allure3")
-def test_unicode_in_docstring_description():
-    """Description can also be below the function"""
-    assert 42 == int(6 * 7)
-
-@pytest.mark.feature("allure4")
-@allure.title("Assert that 2+2=4")
-def test_with_a_title():
-    assert 2 + 2 == 4
-
-@pytest.mark.feature("allure4")
-@allure.title("Dynamic title: {param1} + {param2} = {expected}")
-@pytest.mark.parametrize('param1,param2,expected', [(2, 2, 4),(1, 2, 5)])
-def test_with_parameterized_title(param1, param2, expected):
-    assert param1 + param2 == expected
-
-@pytest.mark.feature("allure4")
-@allure.title("This is a dynamic title that will be replaced")
-def test_with_dynamic_title():
-    assert 2 + 2 == 4
-    allure.dynamic.title('Test completed, used as title')
-
-
-@pytest.mark.feature("allure5")
-def test_with_steps():
-    """Example test case with steps"""
-    with allure.step("Step 1: Initialize variables"):
-        a = 2
-        b = 3
-    
-    with allure.step("Step 2: Perform addition"):
-        result = a + b
-    
-    with allure.step("Step 3: Verify result"):
-        assert result == 5
-
-import tempfile
-import os
-@pytest.mark.feature("allure6")
-def test_with_attachment():
-    """Example test case with attachment"""
-    # Create some data to attach
-    data = "This is sample data for attachment\nLine 2\nLine 3"
-    
-    # Attach text data
-    allure.attach(data, name="Sample Data", attachment_type=allure.attachment_type.TEXT)
-    
-    # Create and attach a simple file
-    with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as f:
-        f.write("Sample file content\nFor testing attachment feature")
-        temp_file_path = f.name
-    
-    # Attach the file
-    allure.attach.file(temp_file_path, name="Attached File", 
-                      attachment_type=allure.attachment_type.TEXT)
-    
-    # Clean up temporary file
-    os.unlink(temp_file_path)
-    
-    assert True
-
-@pytest.mark.feature("allure7")
-def test_mixed_steps_and_attachments():
-    """Example test case combining steps and attachments"""
-    with allure.step("Initialize test data"):
-        test_data = {"name": "John", "age": 30, "city": "New York"}
-        
-    with allure.step("Convert data to JSON string"):
-        import json
-        json_data = json.dumps(test_data, indent=2)
-        allure.attach(json_data, name="JSON Data", attachment_type=allure.attachment_type.JSON)
-        
-    with allure.step("Validate data"):
-        assert test_data["name"] == "John"
-        assert test_data["age"] == 30
-        
-    with allure.step("Create and attach report"):
-        report_content = f"""
-        Test Report
-        ===========
-        Name: {test_data['name']}
-        Age: {test_data['age']}
-        City: {test_data['city']}
-        Status: PASSED
-        """
-        allure.attach(report_content, name="Test Report", 
-                     attachment_type=allure.attachment_type.TEXT)
+def test_config():
+    assert (
+        config_instance.get_nested_config("database.host", "localhost") == "127.0.0.1"
+    )