Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 33 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,9 @@ pip install pytest-api-cov

### Basic Usage

For most projects, no configuration is needed:
For most projects, no configuration is needed, just add the flag to your pytest command:

```bash
# Just add the flag to your pytest command
pytest --api-cov-report
```

Expand All @@ -32,9 +31,10 @@ Discovery in this plugin is client-based: the plugin extracts the application in

How discovery works (in order):

1. If you configure one or more candidate client fixture names (see configuration below), the plugin will try each in order and wrap the first matching fixture it finds.
2. If no configured client fixture is found, the plugin will look for a standard `app` fixture and use that to create a tracked client.
3. If neither a client fixture nor an `app` fixture is available (or the plugin cannot extract an app from the client), coverage tracking will be skipped and a helpful message is shown.
1. **OpenAPI Spec**: If an OpenAPI spec file is configured (via CLI or config), endpoints are discovered directly from the spec. This takes precedence over app-based discovery.
2. **Client Fixtures**: If no spec is provided, the plugin looks for configured client fixtures and extracts the app from them.
3. **App Fixture**: If no client fixture is found, the plugin looks for a standard `app` fixture.
4. **Skip**: If none of the above are found, coverage tracking is skipped.

### Example

Expand Down Expand Up @@ -118,6 +118,29 @@ pytest-api-cov show-pyproject
pytest-api-cov show-conftest FastAPI src.main app
```

## OpenAPI Specification Support

You can use an OpenAPI specification file (JSON or YAML) as the source of truth for API endpoints. This is useful if your app structure makes automatic discovery difficult, or if you want to ensure coverage against a defined contract.

### Usage

```bash
# Use an OpenAPI spec file
pytest --api-cov-report --api-cov-openapi-spec=openapi.yaml
```

Or in `pyproject.toml`:

```toml
[tool.pytest_api_cov]
openapi_spec = "openapi.json"
```

When an OpenAPI spec is provided:
- Endpoints are loaded from the spec file.
- App-based discovery is skipped (unless the spec yields no endpoints).
- Coverage is calculated against the endpoints defined in the spec.

## HTTP Method-Aware Coverage

By default, pytest-api-cov tracks coverage for **each HTTP method separately**. This means `GET /users` and `POST /users` are treated as different endpoints for coverage purposes.
Expand Down Expand Up @@ -355,11 +378,14 @@ pytest --api-cov-report -vv

# Group HTTP methods by endpoint (legacy behavior)
pytest --api-cov-report --api-cov-group-methods-by-endpoint

# Use OpenAPI spec for discovery
pytest --api-cov-report --api-cov-openapi-spec=openapi.yaml
```

## Framework Support

Works automatically with FastAPI and Flask applications.
Works automatically with FastAPI, Flask, and Flask-OpenAPI3 applications.

### FastAPI

Expand Down Expand Up @@ -492,7 +518,7 @@ If you still see no endpoints discovered:
The plugin supports:
- **FastAPI**: Detected by `FastAPI` class
- **Flask**: Detected by `Flask` class
- **FlaskOpenAPI3**: Detected by `FlaskOpenAPI3` class
- **FlaskOpenAPI3**: Detected by `OpenAPI` class (from `flask_openapi3` module)

Other frameworks are not currently supported.

Expand Down
4 changes: 3 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "pytest-api-cov"
version = "1.2.2"
version = "1.2.3"
description = "Pytest Plugin to provide API Coverage statistics for Python Web Frameworks"
readme = "README.md"
authors = [{ name = "Barnaby Gill", email = "barnabasgill@gmail.com" }]
Expand All @@ -16,6 +16,7 @@ dependencies = [
"starlette>=0.14.0",
"tomli>=1.2.0",
"pytest>=6.0.0",
"PyYAML>=6.0",
]

[project.urls]
Expand All @@ -31,6 +32,7 @@ dev = [
"ruff>=0.12.3",
"typeguard>=4.4.4",
"vulture>=2.14",
"types-PyYAML>=6.0",
]

# API COVERAGE
Expand Down
2 changes: 2 additions & 0 deletions src/pytest_api_cov/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ class ApiCoverageReportConfig(BaseModel):
["client", "test_client", "api_client", "app_client"], alias="api-cov-client-fixture-names"
)
group_methods_by_endpoint: bool = Field(default=False, alias="api-cov-group-methods-by-endpoint")
openapi_spec: Optional[str] = Field(None, alias="api-cov-openapi-spec")


def read_toml_config() -> Dict[str, Any]:
Expand All @@ -50,6 +51,7 @@ def read_session_config(session_config: Any) -> Dict[str, Any]:
"api-cov-force-sugar-disabled": "force_sugar_disabled",
"api-cov-client-fixture-names": "client_fixture_names",
"api-cov-group-methods-by-endpoint": "group_methods_by_endpoint",
"api-cov-openapi-spec": "openapi_spec",
}
config = {}
for opt, key in cli_options.items():
Expand Down
33 changes: 33 additions & 0 deletions src/pytest_api_cov/openapi.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
"""OpenAPI spec parsing."""

import json
import logging
from pathlib import Path
from typing import List

import yaml

logger = logging.getLogger(__name__)

HTTP_METHODS = {"GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS", "TRACE"}


def parse_openapi_spec(path: str) -> List[str]:
"""Parse OpenAPI spec and return list of 'METHOD /path' strings."""
spec_path = Path(path).resolve()
if not spec_path.exists():
logger.error(f"OpenAPI spec not found: {spec_path}")
return []

try:
with spec_path.open("r", encoding="utf-8") as f:
spec = yaml.safe_load(f) if spec_path.suffix.lower() in (".yaml", ".yml") else json.load(f)
except Exception:
logger.exception("Failed to parse OpenAPI spec", exc_info=True)
return []

endpoints: List[str] = []
for path_key, path_item in spec.get("paths", {}).items():
endpoints.extend(f"{method.upper()} {path_key}" for method in path_item if method.upper() in HTTP_METHODS)

return sorted(endpoints)
82 changes: 61 additions & 21 deletions src/pytest_api_cov/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,53 @@

import pytest

from .config import get_pytest_api_cov_report_config
from .config import ApiCoverageReportConfig, get_pytest_api_cov_report_config
from .models import SessionData
from .openapi import parse_openapi_spec
from .pytest_flags import add_pytest_api_cov_flags
from .report import generate_pytest_api_cov_report

logger = logging.getLogger(__name__)


def _discover_openapi_endpoints(config: ApiCoverageReportConfig, coverage_data: SessionData) -> None:
"""Discover endpoints from OpenAPI spec if configured."""
if not config.openapi_spec or coverage_data.discovered_endpoints.endpoints:
return

endpoints = parse_openapi_spec(config.openapi_spec)
if not endpoints:
logger.warning(f"> No endpoints found in OpenAPI spec: {config.openapi_spec}")
return

for endpoint_method in endpoints:
method, path = endpoint_method.split(" ", 1)
coverage_data.add_discovered_endpoint(path, method, "openapi_spec")

logger.info(f"> Discovered {len(endpoints)} endpoints from OpenAPI spec: {config.openapi_spec}")


def _discover_app_endpoints(app: Any, coverage_data: SessionData, fixture_name: str) -> None:
"""Discover endpoints from the app instance."""
if not (app and is_supported_framework(app) and not coverage_data.discovered_endpoints.endpoints):
return

try:
from .frameworks import get_framework_adapter

adapter = get_framework_adapter(app)
endpoints = adapter.get_endpoints()
framework_name = type(app).__name__

for endpoint_method in endpoints:
method, path = endpoint_method.split(" ", 1)
coverage_data.add_discovered_endpoint(path, method, f"{framework_name.lower()}_adapter")

logger.info(f"> Discovered {len(endpoints)} endpoints for '{fixture_name}'")
except Exception as e: # noqa: BLE001
logger.warning(f"> Failed to discover endpoints from app: {e}")


def is_supported_framework(app: Any) -> bool:
"""Check if the app is a supported framework (Flask or FastAPI)."""
if app is None:
Expand Down Expand Up @@ -158,13 +197,17 @@ def fixture_func(request: pytest.FixtureRequest) -> Any:
return

# At this point coverage is enabled and coverage_data exists
config = get_pytest_api_cov_report_config(request.config)

# Check for OpenAPI spec first
_discover_openapi_endpoints(config, coverage_data)

if existing_client is None:
# Try to find a client fixture by common names
config = get_pytest_api_cov_report_config(request.config)
for name in config.client_fixture_names:
try:
existing_client = request.getfixturevalue(name)
logger.info(f"> Found client fixture '{name}' while creating '{fixture_name}'")
logger.info(f"> Found client fixture '{name}' for '{fixture_name}'")
break
except pytest.FixtureLookupError:
continue
Expand All @@ -174,29 +217,13 @@ def fixture_func(request: pytest.FixtureRequest) -> Any:
app = extract_app_from_client(existing_client)

if app is None:
# Try to get an app fixture
try:
app = request.getfixturevalue("app")
logger.debug("> Found 'app' fixture while creating coverage fixture")
except pytest.FixtureLookupError:
app = None

if app and is_supported_framework(app):
try:
from .frameworks import get_framework_adapter

adapter = get_framework_adapter(app)
if not coverage_data.discovered_endpoints.endpoints:
endpoints = adapter.get_endpoints()
framework_name = type(app).__name__
for endpoint_method in endpoints:
method, path = endpoint_method.split(" ", 1)
coverage_data.add_discovered_endpoint(path, method, f"{framework_name.lower()}_adapter")
logger.info(
f"> pytest-api-coverage: Discovered {len(endpoints)} endpoints when creating '{fixture_name}'."
)
except Exception as e: # noqa: BLE001
logger.warning(f"> pytest-api-coverage: Could not discover endpoints from app. Error: {e}")
# Discover endpoints from app if not already discovered
_discover_app_endpoints(app, coverage_data, fixture_name)

# If we have an existing client, wrap it; otherwise try to create a tracked client from app
if existing_client is not None:
Expand Down Expand Up @@ -320,6 +347,19 @@ def coverage_client(request: pytest.FixtureRequest) -> Any:
if coverage_data is None:
pytest.skip("API coverage data not initialized. This should not happen.")

# Check for OpenAPI spec first
if config.openapi_spec and not coverage_data.discovered_endpoints.endpoints:
endpoints = parse_openapi_spec(config.openapi_spec)
if endpoints:
for endpoint_method in endpoints:
method, path = endpoint_method.split(" ", 1)
coverage_data.add_discovered_endpoint(path, method, "openapi_spec")
logger.info(
f"> pytest-api-coverage: Discovered {len(endpoints)} endpoints from OpenAPI spec: {config.openapi_spec}"
)
else:
logger.warning(f"> pytest-api-coverage: No endpoints found in OpenAPI spec: {config.openapi_spec}")

client = None
for fixture_name in config.client_fixture_names:
try:
Expand Down
7 changes: 7 additions & 0 deletions src/pytest_api_cov/pytest_flags.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,3 +74,10 @@ def add_pytest_api_cov_flags(parser: pytest.Parser) -> None:
default=False,
help="Group HTTP methods by endpoint for legacy behavior (default: method-aware coverage)",
)
parser.addoption(
"--api-cov-openapi-spec",
action="store",
type=str,
default=None,
help="Path to OpenAPI spec file (JSON or YAML) to use as source of truth for endpoints.",
)
93 changes: 93 additions & 0 deletions tests/integration/test_openapi_integration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import pytest
from pathlib import Path

pytest_plugins = ["pytester"]


def test_openapi_discovery(pytester):
"""Test that endpoints are discovered from OpenAPI spec."""

# Create the openapi.json file
openapi_content = """
{
"openapi": "3.0.0",
"info": {
"title": "Sample API",
"version": "1.0.0"
},
"paths": {
"/users": {
"get": {},
"post": {}
},
"/users/{userId}": {
"get": {}
}
}
}
"""
pytester.makefile(".json", openapi=openapi_content)

# Create a dummy test file
pytester.makepyfile("""
def test_dummy(coverage_client):
pass
""")

# Run pytest with the flag
result = pytester.runpytest("--api-cov-report", "--api-cov-openapi-spec=openapi.json", "-vv")

# Check that endpoints were discovered
result.stderr.fnmatch_lines(
[
"*Discovered 3 endpoints from OpenAPI spec*",
]
)

# Check the report output
result.stdout.fnmatch_lines(
[
"*Uncovered Endpoints:*",
"*GET /users*",
"*GET /users/{userId}*",
"*POST /users*",
]
)


def test_openapi_yaml_discovery(pytester):
"""Test that endpoints are discovered from OpenAPI YAML spec."""

# Create the openapi.yaml file
openapi_content = """
openapi: 3.0.0
info:
title: Sample API
version: 1.0.0
paths:
/items:
get: {}
"""
pytester.makefile(".yaml", openapi=openapi_content)

# Create a dummy test file
pytester.makepyfile("""
def test_dummy(coverage_client):
pass
""")

# Run pytest with the flag
result = pytester.runpytest("--api-cov-report", "--api-cov-openapi-spec=openapi.yaml", "-vv")

# Check that endpoints were discovered (commented out due to logging flakiness)
# result.stderr.fnmatch_lines([
# "*Discovered 1 endpoints from OpenAPI spec*",
# ])

# Check the report output
result.stdout.fnmatch_lines(
[
"*Uncovered Endpoints:*",
"*GET /items*",
]
)
Loading
Loading