diff --git a/benchmarks/coverage_fibonacci/config.yaml b/benchmarks/coverage_fibonacci/config.yaml new file mode 100644 index 00000000000..20a85bbe3c4 --- /dev/null +++ b/benchmarks/coverage_fibonacci/config.yaml @@ -0,0 +1,13 @@ +# Coverage benchmark configurations for fibonacci code +# Tests sys.monitoring.DISABLE optimization performance + +small: &base + fib_n_recursive: 10 + +medium: + <<: *base + fib_n_recursive: 15 + +large: + <<: *base + fib_n_recursive: 20 diff --git a/benchmarks/coverage_fibonacci/scenario.py b/benchmarks/coverage_fibonacci/scenario.py new file mode 100644 index 00000000000..3853938ce2a --- /dev/null +++ b/benchmarks/coverage_fibonacci/scenario.py @@ -0,0 +1,51 @@ +""" +Benchmark for coverage collection on recursive code. + +This benchmark ensures that the sys.monitoring.DISABLE optimization +doesn't regress. The DISABLE return value prevents the handler from being +called repeatedly for the same line in recursive functions and loops. + +Without DISABLE: Handler called on every line execution +With DISABLE: Handler called once per unique line +""" + +from typing import Callable +from typing import Generator + +import bm + + +class CoverageFibonacci(bm.Scenario): + """ + Benchmark coverage collection performance on recursive and iterative code. + + Tests the DISABLE optimization: returning sys.monitoring.DISABLE prevents + the handler from being called repeatedly for the same line. + """ + + fib_n_recursive: int + + def run(self) -> Generator[Callable[[int], None], None, None]: + import os + from pathlib import Path + + from ddtrace.internal.coverage.code import ModuleCodeCollector + from ddtrace.internal.coverage.installer import install + + # Install coverage + install(include_paths=[Path(os.getcwd())]) + + # Import after installation + from utils import fibonacci_recursive + + def _(loops: int) -> None: + for _ in range(loops): + # Use coverage context to simulate real pytest per-test coverage + with ModuleCodeCollector.CollectInContext(): + # Recursive: Many function calls, same lines executed repeatedly + result = fibonacci_recursive(self.fib_n_recursive) + + # Verify correctness (don't optimize away) + assert result > 0 + + yield _ diff --git a/benchmarks/coverage_fibonacci/utils.py b/benchmarks/coverage_fibonacci/utils.py new file mode 100644 index 00000000000..32f36fb0197 --- /dev/null +++ b/benchmarks/coverage_fibonacci/utils.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + + +def fibonacci_recursive(n): + if n <= 1: + return n + return fibonacci_recursive(n - 1) + fibonacci_recursive(n - 2) diff --git a/ddtrace/internal/coverage/code.py b/ddtrace/internal/coverage/code.py index 37047d349b2..94c26ca9ab5 100644 --- a/ddtrace/internal/coverage/code.py +++ b/ddtrace/internal/coverage/code.py @@ -3,6 +3,7 @@ from copy import deepcopy from inspect import getmodule import os +import sys from types import CodeType from types import ModuleType import typing as t @@ -225,6 +226,11 @@ def __enter__(self): if self.is_import_coverage: ctx_is_import_coverage.set(self.is_import_coverage) + # For Python 3.12+, re-enable monitoring that was disabled by previous contexts + # This ensures each test/suite gets accurate coverage data + if sys.version_info >= (3, 12): + sys.monitoring.restart_events() + return self def __exit__(self, *args, **kwargs): diff --git a/ddtrace/internal/coverage/instrumentation_py3_12.py b/ddtrace/internal/coverage/instrumentation_py3_12.py index 8cb83616fe3..9b385e3c2d6 100644 --- a/ddtrace/internal/coverage/instrumentation_py3_12.py +++ b/ddtrace/internal/coverage/instrumentation_py3_12.py @@ -21,10 +21,26 @@ RETURN_CONST = dis.opmap["RETURN_CONST"] EMPTY_MODULE_BYTES = bytes([RESUME, 0, RETURN_CONST, 0]) +# Store: (hook, path, import_names_by_line) _CODE_HOOKS: t.Dict[CodeType, t.Tuple[HookType, str, t.Dict[int, t.Tuple[str, t.Optional[t.Tuple[str]]]]]] = {} def instrument_all_lines(code: CodeType, hook: HookType, path: str, package: str) -> t.Tuple[CodeType, CoverageLines]: + """ + Instrument code for coverage tracking using Python 3.12's monitoring API. + + Args: + code: The code object to instrument + hook: The hook function to call + path: The file path + package: The package name + + Note: Python 3.12+ uses an optimized approach where each line callback returns DISABLE + after recording. This means: + - Each line is only reported once per coverage context (test/suite) + - No overhead for repeated line executions (e.g., in loops) + - Full line-by-line coverage data is captured + """ coverage_tool = sys.monitoring.get_tool(sys.monitoring.COVERAGE_ID) if coverage_tool is not None and coverage_tool != "datadog": log.debug("Coverage tool '%s' already registered, not gathering coverage", coverage_tool) @@ -37,10 +53,21 @@ def instrument_all_lines(code: CodeType, hook: HookType, path: str, package: str return _instrument_all_lines_with_monitoring(code, hook, path, package) -def _line_event_handler(code: CodeType, line: int) -> t.Any: - hook, path, import_names = _CODE_HOOKS[code] +def _line_event_handler(code: CodeType, line: int) -> t.Literal[sys.monitoring.DISABLE]: + hook_data = _CODE_HOOKS.get(code) + if hook_data is None: + return sys.monitoring.DISABLE + + hook, path, import_names = hook_data + + # Report the line and then disable monitoring for this specific line + # This ensures each line is only reported once per context, even if executed multiple times (e.g., in loops) import_name = import_names.get(line, None) - return hook((line, path, import_name)) + hook((line, path, import_name)) + + # Return DISABLE to prevent future callbacks for this specific line + # This provides full line coverage with minimal overhead + return sys.monitoring.DISABLE def _register_monitoring(): diff --git a/releasenotes/notes/fix-civisibility-coverage-3-12-e9b6408d8a5dc886.yaml b/releasenotes/notes/fix-civisibility-coverage-3-12-e9b6408d8a5dc886.yaml new file mode 100644 index 00000000000..5a3a2d4f7a7 --- /dev/null +++ b/releasenotes/notes/fix-civisibility-coverage-3-12-e9b6408d8a5dc886.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + CI Visibility: This fix resolves performance issue affecting coverage collection for Python 3.12+ diff --git a/tests/coverage/included_path/constants_dynamic.py b/tests/coverage/included_path/constants_dynamic.py new file mode 100644 index 00000000000..2a8e250ae9e --- /dev/null +++ b/tests/coverage/included_path/constants_dynamic.py @@ -0,0 +1,5 @@ +"""Constants module - imported dynamically""" + +# Module-level constants +OFFSET = 10 +MULTIPLIER = 2 diff --git a/tests/coverage/included_path/constants_toplevel.py b/tests/coverage/included_path/constants_toplevel.py new file mode 100644 index 00000000000..78e82c23850 --- /dev/null +++ b/tests/coverage/included_path/constants_toplevel.py @@ -0,0 +1,6 @@ +"""Constants module - imported at top level""" + +# Module-level constants +MAX_VALUE = 100 +MIN_VALUE = 0 +DEFAULT_MULTIPLIER = 3 diff --git a/tests/coverage/included_path/layer2_dynamic.py b/tests/coverage/included_path/layer2_dynamic.py new file mode 100644 index 00000000000..7c02f7a7017 --- /dev/null +++ b/tests/coverage/included_path/layer2_dynamic.py @@ -0,0 +1,16 @@ +"""Layer 2 - Imported dynamically, has its own imports""" + +# Top-level import even though this module itself is imported dynamically +from tests.coverage.included_path.layer3_toplevel import layer3_toplevel_function + + +def layer2_dynamic_function(b): + # Use top-level import + step1 = layer3_toplevel_function(b) + + # Dynamic imports - both function and constants + from tests.coverage.included_path.constants_dynamic import OFFSET + from tests.coverage.included_path.layer3_dynamic import layer3_dynamic_function + + step2 = layer3_dynamic_function(step1) + return step2 + OFFSET - 5 diff --git a/tests/coverage/included_path/layer2_toplevel.py b/tests/coverage/included_path/layer2_toplevel.py new file mode 100644 index 00000000000..4d4a7512312 --- /dev/null +++ b/tests/coverage/included_path/layer2_toplevel.py @@ -0,0 +1,16 @@ +"""Layer 2 - Has top-level import and dynamic import""" + +# Top-level imports - both function and constants +from tests.coverage.included_path.constants_toplevel import DEFAULT_MULTIPLIER +from tests.coverage.included_path.layer3_toplevel import layer3_toplevel_function + + +def layer2_toplevel_function(a): + # Use the top-level imported function and constant + intermediate = layer3_toplevel_function(a) * DEFAULT_MULTIPLIER + + # Dynamic import inside function + from tests.coverage.included_path.layer3_dynamic import layer3_dynamic_function + + final = layer3_dynamic_function(intermediate) + return final diff --git a/tests/coverage/included_path/layer3_dynamic.py b/tests/coverage/included_path/layer3_dynamic.py new file mode 100644 index 00000000000..09895a600f9 --- /dev/null +++ b/tests/coverage/included_path/layer3_dynamic.py @@ -0,0 +1,6 @@ +"""Layer 3 - Deepest level, imported dynamically""" + + +def layer3_dynamic_function(y): + computed = y + 10 + return computed * 2 diff --git a/tests/coverage/included_path/layer3_toplevel.py b/tests/coverage/included_path/layer3_toplevel.py new file mode 100644 index 00000000000..15eed867849 --- /dev/null +++ b/tests/coverage/included_path/layer3_toplevel.py @@ -0,0 +1,6 @@ +"""Layer 3 - Deepest level with only top-level code""" + + +def layer3_toplevel_function(x): + result = x * 3 + return result diff --git a/tests/coverage/included_path/nested_fixture.py b/tests/coverage/included_path/nested_fixture.py new file mode 100644 index 00000000000..10d33244e8f --- /dev/null +++ b/tests/coverage/included_path/nested_fixture.py @@ -0,0 +1,33 @@ +""" +Fixture code with complex nested imports. + +This fixture has: +- Top-level imports +- Dynamic (function-level) imports +And the imported modules themselves have more imports (both top-level and dynamic) +""" + +# Top-level imports +from tests.coverage.included_path.layer2_toplevel import layer2_toplevel_function + + +def fixture_toplevel_path(value): + """Uses top-level imported function""" + result = layer2_toplevel_function(value) + return result + + +def fixture_dynamic_path(value): + """Uses dynamically imported function""" + # Dynamic import at function level + from tests.coverage.included_path.layer2_dynamic import layer2_dynamic_function + + result = layer2_dynamic_function(value) + return result + + +def fixture_mixed_path(value): + """Uses both paths""" + result1 = fixture_toplevel_path(value) + result2 = fixture_dynamic_path(value) + return result1 + result2 diff --git a/tests/coverage/included_path/reinstrumentation_test_module.py b/tests/coverage/included_path/reinstrumentation_test_module.py new file mode 100644 index 00000000000..46afd72d64d --- /dev/null +++ b/tests/coverage/included_path/reinstrumentation_test_module.py @@ -0,0 +1,39 @@ +""" +Simple test module for testing coverage re-instrumentation across contexts. + +This module provides simple, predictable functions with known line numbers +to help test that coverage collection works correctly across multiple contexts. +""" + + +def simple_function(x, y): + """A simple function with a few lines.""" + result = x + y + return result + + +def function_with_loop(n): + """A function with a loop to test repeated line execution.""" + total = 0 + for i in range(n): + total += i + return total + + +def function_with_branches(condition): + """A function with branches to test different code paths.""" + if condition: + result = "true_branch" + else: + result = "false_branch" + return result + + +def multi_line_function(a, b, c): + """A function with multiple lines to test comprehensive coverage.""" + step1 = a + b + step2 = step1 * c + step3 = step2 - a + step4 = step3 / (b if b != 0 else 1) + result = step4**2 + return result diff --git a/tests/coverage/test_constants_import_tracking.py b/tests/coverage/test_constants_import_tracking.py new file mode 100644 index 00000000000..a68f5f93cb8 --- /dev/null +++ b/tests/coverage/test_constants_import_tracking.py @@ -0,0 +1,195 @@ +""" +Tests for import-time coverage tracking of constant-only modules. + +These tests verify that modules containing only constants (no executable functions) +are properly tracked in import-time coverage, which is important for the Intelligent +Test Runner to understand code dependencies. +""" + +import sys + +import pytest + + +@pytest.mark.subprocess +def test_constants_module_toplevel_import_tracked(): + """ + Test that constant-only modules imported at top-level are tracked in import-time coverage. + + This verifies that even modules with no executable code (only constant declarations) + appear in the import-time dependency tracking. + """ + import os + from pathlib import Path + + from ddtrace.internal.coverage.code import ModuleCodeCollector + from ddtrace.internal.coverage.installer import install + from tests.coverage.utils import _get_relpath_dict + + cwd_path = os.getcwd() + include_path = Path(cwd_path + "/tests/coverage/included_path/") + + install(include_paths=[include_path], collect_import_time_coverage=True) + + # Import module that has top-level constant imports + from tests.coverage.included_path.layer2_toplevel import layer2_toplevel_function + + ModuleCodeCollector.start_coverage() + result = layer2_toplevel_function(5) + ModuleCodeCollector.stop_coverage() + + assert result == 110 # Verify the function works correctly + + # Get coverage with and without imports + covered = _get_relpath_dict( + cwd_path, ModuleCodeCollector._instance._get_covered_lines(include_imported=False) # type: ignore[union-attr] + ) + covered_with_imports = _get_relpath_dict( + cwd_path, ModuleCodeCollector._instance._get_covered_lines(include_imported=True) # type: ignore[union-attr] + ) + + # Verify runtime coverage (without imports) + assert "tests/coverage/included_path/layer2_toplevel.py" in covered + assert "tests/coverage/included_path/layer3_toplevel.py" in covered + + # CRITICAL: Verify import-time coverage includes the constants module + # Even though constants_toplevel.py has no executable code, it should appear + # in import-time dependencies because layer2_toplevel imports from it + assert "tests/coverage/included_path/constants_toplevel.py" in covered_with_imports, ( + "constants_toplevel.py missing from import-time coverage! " + "Constant-only modules should be tracked as dependencies." + ) + + # The constants module should have its lines tracked + constants_lines = covered_with_imports.get("tests/coverage/included_path/constants_toplevel.py", set()) + # Verify it includes the constant declarations (lines 4, 5, 6) + expected_constant_lines = {4, 5, 6} + assert expected_constant_lines.issubset(constants_lines), ( + f"Expected constant declaration lines {expected_constant_lines} in coverage, " + f"but got: {sorted(constants_lines)}" + ) + + +@pytest.mark.subprocess +def test_constants_module_dynamic_import_tracked(): + """ + Test that constant-only modules imported dynamically are tracked in import-time coverage. + + This verifies that dynamically imported constant modules also appear in + import-time dependency tracking. + """ + import os + from pathlib import Path + + from ddtrace.internal.coverage.code import ModuleCodeCollector + from ddtrace.internal.coverage.installer import install + from tests.coverage.utils import _get_relpath_dict + + cwd_path = os.getcwd() + include_path = Path(cwd_path + "/tests/coverage/included_path/") + + install(include_paths=[include_path], collect_import_time_coverage=True) + + # Import module that has dynamic constant imports + from tests.coverage.included_path.layer2_dynamic import layer2_dynamic_function + + ModuleCodeCollector.start_coverage() + result = layer2_dynamic_function(5) + ModuleCodeCollector.stop_coverage() + + assert result == 55 # Verify the function works correctly + + # Get coverage with and without imports + covered = _get_relpath_dict( + cwd_path, ModuleCodeCollector._instance._get_covered_lines(include_imported=False) # type: ignore[union-attr] + ) + covered_with_imports = _get_relpath_dict( + cwd_path, ModuleCodeCollector._instance._get_covered_lines(include_imported=True) # type: ignore[union-attr] + ) + + # Verify runtime coverage (without imports) + assert "tests/coverage/included_path/layer2_dynamic.py" in covered + + # CRITICAL: Verify import-time coverage includes the dynamically imported constants module + assert "tests/coverage/included_path/constants_dynamic.py" in covered_with_imports, ( + "constants_dynamic.py missing from import-time coverage! " + "Dynamically imported constant-only modules should be tracked as dependencies." + ) + + # The constants module should have its lines tracked + constants_lines = covered_with_imports.get("tests/coverage/included_path/constants_dynamic.py", set()) + # Verify it includes the constant declarations (lines 4, 5) + expected_constant_lines = {4, 5} + assert expected_constant_lines.issubset(constants_lines), ( + f"Expected constant declaration lines {expected_constant_lines} in coverage, " + f"but got: {sorted(constants_lines)}" + ) + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Test specific to Python 3.12+ monitoring API") +@pytest.mark.subprocess +def test_constants_module_reinstrumentation(): + """ + Test that constant-only modules are properly re-instrumented between coverage collections. + + This ensures that constant modules appear consistently in import-time coverage + across multiple start/stop cycles (important for per-test coverage in pytest). + """ + import os + from pathlib import Path + + from ddtrace.internal.coverage.code import ModuleCodeCollector + from ddtrace.internal.coverage.installer import install + from tests.coverage.utils import _get_relpath_dict + + cwd_path = os.getcwd() + include_path = Path(cwd_path + "/tests/coverage/included_path/") + + install(include_paths=[include_path], collect_import_time_coverage=True) + + from tests.coverage.included_path.layer2_toplevel import layer2_toplevel_function + + # First coverage collection + ModuleCodeCollector.start_coverage() + layer2_toplevel_function(5) + ModuleCodeCollector.stop_coverage() + + first_covered_with_imports = _get_relpath_dict( + cwd_path, ModuleCodeCollector._instance._get_covered_lines(include_imported=True) # type: ignore[union-attr] + ) + + # Clear coverage to simulate new test + ModuleCodeCollector._instance.covered.clear() # type: ignore[union-attr] + + # Second coverage collection + ModuleCodeCollector.start_coverage() + layer2_toplevel_function(10) + ModuleCodeCollector.stop_coverage() + + second_covered_with_imports = _get_relpath_dict( + cwd_path, ModuleCodeCollector._instance._get_covered_lines(include_imported=True) # type: ignore[union-attr] + ) + + # CRITICAL: Both collections should track the constants module + assert ( + "tests/coverage/included_path/constants_toplevel.py" in first_covered_with_imports + ), "First collection missing constants_toplevel.py" + assert ( + "tests/coverage/included_path/constants_toplevel.py" in second_covered_with_imports + ), "Second collection missing constants_toplevel.py - re-instrumentation failed for constant modules!" + + # Both should have the same lines for the constants module + first_constants = first_covered_with_imports["tests/coverage/included_path/constants_toplevel.py"] + second_constants = second_covered_with_imports["tests/coverage/included_path/constants_toplevel.py"] + + assert first_constants == second_constants, ( + f"Constants coverage differs between collections - re-instrumentation issue!\n" + f" First: {sorted(first_constants)}\n" + f" Second: {sorted(second_constants)}" + ) + + # Verify the constants are actually tracked + expected_lines = {4, 5, 6} + assert expected_lines.issubset( + second_constants + ), f"Expected constant lines {expected_lines} in second collection, got: {sorted(second_constants)}" diff --git a/tests/coverage/test_coverage_context_reinstrumentation.py b/tests/coverage/test_coverage_context_reinstrumentation.py new file mode 100644 index 00000000000..7e30a886308 --- /dev/null +++ b/tests/coverage/test_coverage_context_reinstrumentation.py @@ -0,0 +1,306 @@ +""" +Regression tests for Python 3.12+ coverage re-instrumentation between contexts. + +These tests verify that coverage collection properly re-instruments code between +different coverage contexts (e.g., between tests or suites). This is critical +for the DISABLE optimization in Python 3.12+ where monitoring is disabled after +each line is recorded, and must be re-enabled for subsequent contexts. + +The tests are intentionally high-level to survive implementation changes while +ensuring: +1. Each context gets complete coverage data +2. No coverage gaps occur between contexts +3. Code executed in multiple contexts is properly tracked in each +4. Loops and repeated execution don't prevent coverage in new contexts +""" + +import sys + +import pytest + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Test specific to Python 3.12+ monitoring API") +@pytest.mark.subprocess +def test_nested_contexts_maintain_independence(): + """ + Test that nested coverage contexts maintain independence and proper re-instrumentation. + + This ensures the context stack properly handles re-instrumentation when entering + nested contexts. + + IMPORTANT NOTE: The overlapping coverage does not get tracked by the outer context + """ + import os + from pathlib import Path + + from ddtrace.internal.coverage.code import ModuleCodeCollector + from ddtrace.internal.coverage.installer import install + from tests.coverage.utils import _get_relpath_dict + + cwd_path = os.getcwd() + include_path = Path(cwd_path + "/tests/coverage/included_path/") + + install(include_paths=[include_path]) + + from tests.coverage.included_path.callee import called_in_context_main + from tests.coverage.included_path.callee import called_in_session_main + + # Outer context + with ModuleCodeCollector.CollectInContext() as outer_context: + called_in_session_main(1, 2) + + # Inner nested context - should capture everything independently + with ModuleCodeCollector.CollectInContext() as inner_context: + called_in_context_main(3, 4) + inner_covered = _get_relpath_dict(cwd_path, inner_context.get_covered_lines()) + + # Execute more code in outer context after inner completes + called_in_context_main(3, 4) # NOTE: This is not tracked as overlaps with inner + outer_covered = _get_relpath_dict(cwd_path, outer_context.get_covered_lines()) + + # Inner context should have captured its specific execution + expected_inner = { + "tests/coverage/included_path/callee.py": {10, 11, 13, 14}, + "tests/coverage/included_path/in_context_lib.py": {1, 2, 5}, + } + expected_outer = { + "tests/coverage/included_path/callee.py": {2, 3, 5, 6}, + "tests/coverage/included_path/lib.py": {1, 2, 5}, + } + + # Inner context should have complete coverage for its execution + assert ( + inner_covered == expected_inner + ), f"Inner context coverage mismatch: expected={expected_inner} vs actual={inner_covered}" + + # Inner context should have complete coverage for its execution + assert ( + outer_covered == expected_outer + ), f"Inner context coverage mismatch: expected={expected_outer} vs actual={outer_covered}" + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Test specific to Python 3.12+ monitoring API") +@pytest.mark.subprocess +def test_many_sequential_contexts_no_degradation(): + """ + Test that coverage quality doesn't degrade over many sequential contexts. + + This is a stress test to ensure the re-instrumentation mechanism works + consistently across many contexts without accumulating issues. + """ + import os + from pathlib import Path + + from ddtrace.internal.coverage.code import ModuleCodeCollector + from ddtrace.internal.coverage.installer import install + from tests.coverage.utils import _get_relpath_dict + + cwd_path = os.getcwd() + include_path = Path(cwd_path + "/tests/coverage/included_path/") + + install(include_paths=[include_path]) + + from tests.coverage.included_path.callee import called_in_session_main + + # Collect coverage from multiple sequential contexts + all_context_coverages = [] + + for i in range(3): + with ModuleCodeCollector.CollectInContext() as context: + called_in_session_main(i, i + 1) + context_covered = _get_relpath_dict(cwd_path, context.get_covered_lines()) + all_context_coverages.append(context_covered) + + # Expected coverage for callee.py - the runtime execution lines + expected_callee_lines = {2, 3, 5, 6} + + # Verify all contexts got the same coverage for callee.py + for idx, context_covered in enumerate(all_context_coverages): + assert "tests/coverage/included_path/callee.py" in context_covered, f"Context {idx} missing callee.py" + + # Check callee.py lines match (these are runtime, not import-time) + actual_callee = context_covered["tests/coverage/included_path/callee.py"] + if idx == 0: + # First context includes import lines + assert expected_callee_lines.issubset(actual_callee), f"Context {idx} missing expected callee lines" + else: + # Subsequent contexts should have at least the runtime lines + assert expected_callee_lines.issubset(actual_callee), f"Context {idx} missing expected callee lines" + + # Check lib.py exists and has line 2 (the function body) + assert "tests/coverage/included_path/lib.py" in context_covered, f"Context {idx} missing lib.py" + assert ( + 2 in context_covered["tests/coverage/included_path/lib.py"] + ), f"Context {idx} missing lib.py line 2 - re-instrumentation failed!" + + # Critical: Coverage should not decrease over iterations + # All contexts should have the same runtime lines for callee.py + first_callee = all_context_coverages[0].get("tests/coverage/included_path/callee.py", set()) + last_callee = all_context_coverages[-1].get("tests/coverage/included_path/callee.py", set()) + + # Check that expected_callee_lines are in both first and last + assert expected_callee_lines.issubset(first_callee) and expected_callee_lines.issubset( + last_callee + ), f"Coverage degraded: first had {first_callee}, last had {last_callee}" + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Test specific to Python 3.12+ monitoring API") +@pytest.mark.subprocess +def test_context_after_session_coverage(): + """ + Test that context-based coverage works correctly after session-level coverage. + + This ensures that transitioning from session coverage to context coverage + properly re-instruments the code. + """ + import os + from pathlib import Path + + from ddtrace.internal.coverage.code import ModuleCodeCollector + from ddtrace.internal.coverage.installer import install + from tests.coverage.utils import _get_relpath_dict + + cwd_path = os.getcwd() + include_path = Path(cwd_path + "/tests/coverage/included_path/") + + install(include_paths=[include_path]) + + from tests.coverage.included_path.callee import called_in_context_main + from tests.coverage.included_path.callee import called_in_session_main + + # Session-level coverage + ModuleCodeCollector.start_coverage() + called_in_session_main(1, 2) + ModuleCodeCollector.stop_coverage() + + session_covered = _get_relpath_dict(cwd_path, ModuleCodeCollector._instance._get_covered_lines()) # type: ignore[union-attr] + + # Now use context-based coverage - should still get complete coverage + with ModuleCodeCollector.CollectInContext() as context1: + called_in_session_main(3, 4) + called_in_context_main(5, 6) + context1_covered = _get_relpath_dict(cwd_path, context1.get_covered_lines()) + + # Another context - should also get complete coverage + with ModuleCodeCollector.CollectInContext() as context2: + called_in_session_main(7, 8) + called_in_context_main(9, 10) + context2_covered = _get_relpath_dict(cwd_path, context2.get_covered_lines()) + + # Session should have captured called_in_session_main (runtime lines) + expected_session_runtime = {2, 3, 5, 6} + + # Contexts should have both functions (runtime lines) + expected_context_callee_runtime = {2, 3, 5, 6, 10, 11, 13, 14} + + # Verify session coverage + assert "tests/coverage/included_path/callee.py" in session_covered + assert expected_session_runtime.issubset(session_covered["tests/coverage/included_path/callee.py"]) + assert 2 in session_covered["tests/coverage/included_path/lib.py"], "Session missing lib.py line 2" + + # Verify context 1 coverage + assert "tests/coverage/included_path/callee.py" in context1_covered + assert expected_context_callee_runtime.issubset(context1_covered["tests/coverage/included_path/callee.py"]) + assert 2 in context1_covered["tests/coverage/included_path/lib.py"], "Context 1 missing lib.py line 2" + assert ( + 2 in context1_covered["tests/coverage/included_path/in_context_lib.py"] + ), "Context 1 missing in_context_lib.py line 2" + + # Verify context 2 coverage + assert "tests/coverage/included_path/callee.py" in context2_covered + assert expected_context_callee_runtime.issubset(context2_covered["tests/coverage/included_path/callee.py"]) + assert ( + 2 in context2_covered["tests/coverage/included_path/lib.py"] + ), "Context 2 missing lib.py line 2 - re-instrumentation failed!" + assert ( + 2 in context2_covered["tests/coverage/included_path/in_context_lib.py"] + ), "Context 2 missing in_context_lib.py line 2 - re-instrumentation failed!" + + # Critical: Both contexts should have the same runtime lines for callee.py + context1_callee = context1_covered["tests/coverage/included_path/callee.py"] + context2_callee = context2_covered["tests/coverage/included_path/callee.py"] + + assert expected_context_callee_runtime.issubset(context1_callee) and expected_context_callee_runtime.issubset( + context2_callee + ), ( + f"Context coverages differ - re-instrumentation may have failed: " + f"context1={context1_callee}, context2={context2_callee}" + ) + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Test specific to Python 3.12+ monitoring API") +@pytest.mark.subprocess +def test_comprehensive_reinstrumentation_with_simple_module(): + """ + Comprehensive test using a simple controlled module to verify re-instrumentation. + + This test uses a dedicated test module with predictable line numbers to ensure + re-instrumentation works correctly across various code patterns. + """ + import os + from pathlib import Path + + from ddtrace.internal.coverage.code import ModuleCodeCollector + from ddtrace.internal.coverage.installer import install + from tests.coverage.utils import _get_relpath_dict + + cwd_path = os.getcwd() + include_path = Path(cwd_path + "/tests/coverage/included_path/") + + install(include_paths=[include_path]) + + from tests.coverage.included_path.reinstrumentation_test_module import function_with_branches + from tests.coverage.included_path.reinstrumentation_test_module import function_with_loop + from tests.coverage.included_path.reinstrumentation_test_module import multi_line_function + from tests.coverage.included_path.reinstrumentation_test_module import simple_function + + # Context 1: Execute all functions + with ModuleCodeCollector.CollectInContext() as context1: + simple_function(1, 2) + function_with_loop(5) + function_with_branches(True) + multi_line_function(2, 3, 4) + context1_covered = _get_relpath_dict(cwd_path, context1.get_covered_lines()) + + # Context 2: Execute the same functions with different arguments + with ModuleCodeCollector.CollectInContext() as context2: + simple_function(10, 20) + function_with_loop(10) + function_with_branches(True) + multi_line_function(5, 6, 7) + context2_covered = _get_relpath_dict(cwd_path, context2.get_covered_lines()) + + # Context 3: Execute with different branch paths + with ModuleCodeCollector.CollectInContext() as context3: + simple_function(100, 200) + function_with_loop(3) + function_with_branches(False) # Different branch + multi_line_function(1, 1, 1) + context3_covered = _get_relpath_dict(cwd_path, context3.get_covered_lines()) + + module_path = "tests/coverage/included_path/reinstrumentation_test_module.py" + + # All contexts should have coverage for the module + assert module_path in context1_covered, f"Context 1 missing {module_path}" + assert module_path in context2_covered, f"Context 2 missing {module_path}" + assert module_path in context3_covered, f"Context 3 missing {module_path}" + + # Expected lines for context 1 and 2 (same branch in function_with_branches) + expected_lines_true_branch = {11, 12, 17, 18, 19, 20, 25, 26, 29, 34, 35, 36, 37, 38, 39} + + # Expected lines for context 3 (false branch in function_with_branches) + expected_lines_false_branch = {11, 12, 17, 18, 19, 20, 25, 28, 29, 34, 35, 36, 37, 38, 39} + + # Verify contexts 1 and 2 captured the true branch + assert ( + context1_covered[module_path] == expected_lines_true_branch + ), f"Context 1 coverage mismatch: expected={expected_lines_true_branch} vs actual={context1_covered[module_path]}" + + assert ( + context2_covered[module_path] == expected_lines_true_branch + ), f"Context 2 coverage mismatch: expected={expected_lines_true_branch} vs actual={context2_covered[module_path]}" + + # Verify context 3 captured the false branch + assert ( + context3_covered[module_path] == expected_lines_false_branch + ), f"Context 3 coverage mismatch: expected={expected_lines_false_branch} vs actual={context3_covered[module_path]}" diff --git a/tests/coverage/test_instrumentation_py312_disable.py b/tests/coverage/test_instrumentation_py312_disable.py new file mode 100644 index 00000000000..abf4ac2e577 --- /dev/null +++ b/tests/coverage/test_instrumentation_py312_disable.py @@ -0,0 +1,63 @@ +""" +Unit test for Python 3.12+ instrumentation DISABLE optimization. + +Verifies that _line_event_handler returns sys.monitoring.DISABLE to prevent +repeated callbacks for the same line within a context. +""" +import sys + +import pytest + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Python 3.12+ monitoring API only") +def test_line_event_handler_returns_disable(): + """ + Test that _line_event_handler returns DISABLE after recording a line. + + This is critical for performance - returning DISABLE prevents the monitoring + system from calling the handler repeatedly for the same line (e.g., in loops). + """ + from ddtrace.internal.coverage.instrumentation_py3_12 import _CODE_HOOKS + from ddtrace.internal.coverage.instrumentation_py3_12 import _line_event_handler + + # Create a simple code object and register it + code_obj = compile("x = 1", "", "exec") + + # Track calls to the hook + calls = [] + + def mock_hook(line_info): + calls.append(line_info) + + # Register the code object with our hook + _CODE_HOOKS[code_obj] = (mock_hook, "/test/path.py", {}) + + try: + # Call the handler + result = _line_event_handler(code_obj, 1) + + # CRITICAL: Must return DISABLE to prevent repeated callbacks + assert result == sys.monitoring.DISABLE, f"_line_event_handler must return sys.monitoring.DISABLE, got {result}" + + # Verify the hook was called + assert len(calls) == 1 + assert calls[0] == (1, "/test/path.py", None) + finally: + # Cleanup + if code_obj in _CODE_HOOKS: + del _CODE_HOOKS[code_obj] + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Python 3.12+ monitoring API only") +def test_line_event_handler_returns_disable_for_missing_code(): + """Test that handler returns DISABLE even when code object is missing (graceful error handling).""" + from ddtrace.internal.coverage.instrumentation_py3_12 import _line_event_handler + + # Create a code object that's NOT registered + code_obj = compile("y = 2", "", "exec") + + # Call handler with unregistered code object + result = _line_event_handler(code_obj, 1) + + # Should still return DISABLE (graceful handling) + assert result == sys.monitoring.DISABLE, f"Handler should return DISABLE even for missing code, got {result}" diff --git a/tests/coverage/test_nested_dynamic_imports.py b/tests/coverage/test_nested_dynamic_imports.py new file mode 100644 index 00000000000..985864d61b6 --- /dev/null +++ b/tests/coverage/test_nested_dynamic_imports.py @@ -0,0 +1,221 @@ +""" +Test complex nested import scenarios with multiple layers of top-level and dynamic imports. + +This test checks if re-instrumentation works correctly when: +- Fixture code has top-level imports +- Fixture code has dynamic (function-level) imports +- Those imported modules themselves have more imports (both top-level and dynamic) +- Multiple contexts execute the same code paths + +The fixture modules are in tests/coverage/included_path/: +- nested_fixture.py (main fixture with top-level and dynamic imports) +- layer2_toplevel.py, layer2_dynamic.py (imported by fixture) +- layer3_toplevel.py, layer3_dynamic.py (imported by layer2) +""" + +import sys + +import pytest + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Test specific to Python 3.12+ monitoring API") +@pytest.mark.subprocess +def test_nested_imports_mixed_path_reinstrumentation(): + """ + Test re-instrumentation with nested imports using both top-level and dynamic paths. + + This is the most comprehensive test - it exercises ALL import paths in sequence. + """ + # DEV: Required local imports for subprocess decorator + import os + from pathlib import Path + + from ddtrace.internal.coverage.code import ModuleCodeCollector + from ddtrace.internal.coverage.installer import install + from tests.coverage.utils import _get_relpath_dict + + cwd_path = os.getcwd() + include_path = Path(cwd_path + "/tests/coverage/included_path/") + + install(include_paths=[include_path]) + + from tests.coverage.included_path.nested_fixture import fixture_mixed_path + + # Context 1: Execute all paths + with ModuleCodeCollector.CollectInContext() as context1: + fixture_mixed_path(5) + context1_covered = _get_relpath_dict(cwd_path, context1.get_covered_lines()) + + # Context 2: Execute all paths again + with ModuleCodeCollector.CollectInContext() as context2: + fixture_mixed_path(10) + context2_covered = _get_relpath_dict(cwd_path, context2.get_covered_lines()) + + # Expected runtime lines (captured in all contexts) - mixed path uses BOTH toplevel and dynamic + # Note: constant-only modules don't appear in coverage as they have no executable code + expected_runtime = { + "tests/coverage/included_path/nested_fixture.py": {16, 17, 23, 25, 26, 31, 32, 33}, + "tests/coverage/included_path/layer2_toplevel.py": {10, 13, 15, 16}, + "tests/coverage/included_path/layer2_dynamic.py": {9, 12, 13, 15, 16}, + "tests/coverage/included_path/layer3_toplevel.py": {5, 6}, + "tests/coverage/included_path/layer3_dynamic.py": {5, 6}, + } + + # Expected import-time lines (only in context 1) + expected_import_time = { + "tests/coverage/included_path/layer2_dynamic.py": {1, 4, 7}, # docstring + import + function def + "tests/coverage/included_path/layer3_dynamic.py": {1, 4}, # docstring + function def + } + + for file_path, expected_lines in expected_runtime.items(): + # All contexts should have the file + assert file_path in context1_covered, f"Context 1 missing {file_path}" + assert file_path in context2_covered, f"Context 2 missing {file_path} - re-instrumentation failed!" + + # Check runtime lines are captured in context 1 and 2 + assert context2_covered[file_path] == expected_lines, ( + f"{file_path}: Runtime coverage mismatch\n" + f" Expected: {sorted(expected_lines)}\n" + f" Got: {sorted(context2_covered[file_path])}" + ) + + # Contexts should have runtime + any import-time lines + expected_context = expected_lines | expected_import_time.get(file_path, set()) + assert context1_covered[file_path] == expected_context, ( + f"{file_path}: Context 1 coverage mismatch\n" + f" Expected: {sorted(expected_context)}\n" + f" Got: {sorted(context1_covered[file_path])}" + ) + + for file_path, expected_lines in expected_import_time.items(): + assert not expected_lines.issubset(context2_covered[file_path]), ( + f"{file_path}: Import time not expected in Context 2 coverage\n" f" Got: {expected_lines}" + ) + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Test specific to Python 3.12+ monitoring API") +@pytest.mark.subprocess +def test_nested_imports_interleaved_execution(): + """ + Test re-instrumentation with interleaved execution of different import paths. + + This simulates a realistic scenario where different tests might call different + code paths, and we need to ensure ALL paths are properly instrumented in each context. + """ + # DEV: Required local imports for subprocess decorator + import os + from pathlib import Path + + from ddtrace.internal.coverage.code import ModuleCodeCollector + from ddtrace.internal.coverage.installer import install + from tests.coverage.utils import _get_relpath_dict + + cwd_path = os.getcwd() + include_path = Path(cwd_path + "/tests/coverage/included_path/") + + install(include_paths=[include_path]) + + from tests.coverage.included_path.nested_fixture import fixture_dynamic_path + from tests.coverage.included_path.nested_fixture import fixture_toplevel_path + + # Context 1: Execute toplevel path + with ModuleCodeCollector.CollectInContext() as context1: + fixture_toplevel_path(5) + context1_covered = _get_relpath_dict(cwd_path, context1.get_covered_lines()) + + # Context 2: Execute dynamic path (different path) + with ModuleCodeCollector.CollectInContext() as context2: + fixture_dynamic_path(10) + context2_covered = _get_relpath_dict(cwd_path, context2.get_covered_lines()) + + # Context 3: Execute toplevel path again (back to first path) + with ModuleCodeCollector.CollectInContext() as context3: + fixture_toplevel_path(3) + context3_covered = _get_relpath_dict(cwd_path, context3.get_covered_lines()) + + # Context 4: Execute dynamic path again + with ModuleCodeCollector.CollectInContext() as context4: + fixture_dynamic_path(7) + context4_covered = _get_relpath_dict(cwd_path, context4.get_covered_lines()) + + # Expected coverage for contexts 1 and 3 (both use toplevel path) + # Note: constant-only modules don't appear as they have no executable code + expected_toplevel_runtime = { + "tests/coverage/included_path/nested_fixture.py": {16, 17}, + "tests/coverage/included_path/layer2_toplevel.py": {10, 13, 15, 16}, # Updated + "tests/coverage/included_path/layer3_toplevel.py": {5, 6}, + "tests/coverage/included_path/layer3_dynamic.py": {5, 6}, + } + + # Expected coverage for contexts 2 and 4 (both use dynamic path) + expected_dynamic_runtime = { + "tests/coverage/included_path/nested_fixture.py": {23, 25, 26}, + "tests/coverage/included_path/layer2_dynamic.py": {9, 12, 13, 15, 16}, # Updated + "tests/coverage/included_path/layer3_toplevel.py": {5, 6}, + "tests/coverage/included_path/layer3_dynamic.py": {5, 6}, + } + + # Check toplevel path (contexts 1 and 3) + for file_path, expected_lines in expected_toplevel_runtime.items(): + assert file_path in context1_covered, f"Context 1 missing {file_path}" + assert file_path in context3_covered, f"Context 3 missing {file_path} - re-instrumentation failed!" + + # CRITICAL: Context 3 should have exact runtime coverage + assert context3_covered[file_path] == expected_lines, ( + f"{file_path}: Context 3 runtime mismatch\n" + f" Expected: {sorted(expected_lines)}\n" + f" Got: {sorted(context3_covered[file_path])}" + ) + + # Context 1 may have import-time lines for dynamically imported modules + if file_path == "tests/coverage/included_path/layer3_dynamic.py": + # Context 1 captures import-time + runtime for layer3_dynamic (dynamically imported) + expected_context1 = expected_lines | {1, 4} # docstring + function def + assert context1_covered[file_path] == expected_context1, ( + f"{file_path}: Context 1 mismatch\n" + f" Expected: {sorted(expected_context1)}\n" + f" Got: {sorted(context1_covered[file_path])}" + ) + elif file_path == "tests/coverage/included_path/layer2_toplevel.py": + # layer2_toplevel is imported at fixture top-level, so it's imported before Context 1 + # Therefore, Context 1 won't have its import-time lines + assert context1_covered[file_path] == expected_lines, ( + f"{file_path}: Context 1 mismatch\n" + f" Expected: {sorted(expected_lines)}\n" + f" Got: {sorted(context1_covered[file_path])}" + ) + else: + assert context1_covered[file_path] == expected_lines, ( + f"{file_path}: Context 1 mismatch\n" + f" Expected: {sorted(expected_lines)}\n" + f" Got: {sorted(context1_covered[file_path])}" + ) + + # Check dynamic path (contexts 2 and 4) + for file_path, expected_lines in expected_dynamic_runtime.items(): + assert file_path in context2_covered, f"Context 2 missing {file_path}" + assert file_path in context4_covered, f"Context 4 missing {file_path} - re-instrumentation failed!" + + # CRITICAL: Context 4 should have exact runtime coverage (proves re-instrumentation works) + assert context4_covered[file_path] == expected_lines, ( + f"{file_path}: Context 4 runtime mismatch\n" + f" Expected: {sorted(expected_lines)}\n" + f" Got: {sorted(context4_covered[file_path])}" + ) + + # Context 2 is first to use dynamic path, may have import-time lines + # Note: layer3_dynamic was already imported in Context 1, so Context 2 won't have its import-time + if file_path == "tests/coverage/included_path/layer2_dynamic.py": + # Context 2 captures import-time for layer2_dynamic (first time it's imported) + expected_context2 = expected_lines | {1, 4, 7} # docstring + import + function def + assert context2_covered[file_path] == expected_context2, ( + f"{file_path}: Context 2 mismatch\n" + f" Expected: {sorted(expected_context2)}\n" + f" Got: {sorted(context2_covered[file_path])}" + ) + else: + assert context2_covered[file_path] == expected_lines, ( + f"{file_path}: Context 2 mismatch\n" + f" Expected: {sorted(expected_lines)}\n" + f" Got: {sorted(context2_covered[file_path])}" + )