@@ -486,6 +692,7 @@ function generateHtmlReport({ coverage, coverageItems, meta }) {
// coverageData from server
let coverageData = ${coverageDataJson};
let apiCount = ${apiCount};
+ let negativeMetrics = ${negativeMetricsJson};
// Merge duplicates for display only
function unifyByMethodAndPath(items) {
@@ -532,6 +739,8 @@ function generateHtmlReport({ coverage, coverageItems, meta }) {
renderCoverageChart(${coverage.toFixed(2)});
renderTrendChart();
renderTagChart();
+ renderNegativeTestingChart();
+ renderNegativeTestingMetrics();
renderTable();
};
@@ -662,6 +871,176 @@ function generateHtmlReport({ coverage, coverageItems, meta }) {
});
}
+ // Render negative testing chart
+ function renderNegativeTestingChart() {
+ const ctx = document.getElementById('negativeTestChart').getContext('2d');
+ new Chart(ctx, {
+ type: 'doughnut',
+ data: {
+ labels: ['Positive Tests', 'Negative Tests'],
+ datasets: [{
+ data: [negativeMetrics.positive.coverage, negativeMetrics.negative.coverage],
+ backgroundColor: ['#4caf50', '#ff9800']
+ }]
+ },
+ options: {
+ responsive: true,
+ plugins: {
+ legend: { position: 'bottom' },
+ tooltip: {
+ callbacks: {
+ label: function(context) {
+ const label = context.label || '';
+ const value = context.parsed;
+ const isPositive = context.dataIndex === 0;
+ const counts = isPositive ?
+ \`\${negativeMetrics.positive.matched}/\${negativeMetrics.positive.total}\` :
+ \`\${negativeMetrics.negative.matched}/\${negativeMetrics.negative.total}\`;
+ return \`\${label}: \${value.toFixed(1)}% (\${counts})\`;
+ }
+ }
+ }
+ }
+ }
+ });
+ }
+
+ // Render negative testing metrics
+ function renderNegativeTestingMetrics() {
+ // Error Status Metrics
+ const errorStatusContainer = document.getElementById('errorStatusMetrics');
+ let errorHtml = '
';
+
+ Object.entries(negativeMetrics.errorDistribution).forEach(([statusCode, data]) => {
+ const statusColor = getStatusCodeColor(statusCode);
+ errorHtml += \`
+
+ \${statusCode}
+ \${data.coverage.toFixed(1)}% (\${data.matched}/\${data.total})
+
+ \`;
+ });
+ errorHtml += '
';
+ errorStatusContainer.innerHTML = errorHtml;
+
+ // Quality Score
+ const qualityScoreContainer = document.getElementById('qualityScore');
+ const qualityScore = calculateQualityScore();
+ const scoreColor = getScoreColor(qualityScore);
+ qualityScoreContainer.innerHTML = \`
+
+
+ \${qualityScore.toFixed(0)}
+ / 100
+
+
\${getScoreAssessment(qualityScore)}
+
+ \`;
+
+ // Recommendations
+ const recommendationsContainer = document.getElementById('testingRecommendations');
+ const recommendations = generateTestingRecommendations();
+ let recHtml = '
';
+ if (recommendations.length === 0) {
+ recHtml += '- โ
Good negative test coverage detected!
';
+ } else {
+ recommendations.forEach(rec => {
+ recHtml += \`- โ ๏ธ \${rec}
\`;
+ });
+ }
+ recHtml += '
';
+ recommendationsContainer.innerHTML = recHtml;
+ }
+
+ // Helper function to get status code color
+ function getStatusCodeColor(statusCode) {
+ if (statusCode.startsWith('2')) return '#4caf50'; // Green for 2xx
+ if (statusCode.startsWith('4')) return '#ff9800'; // Orange for 4xx
+ if (statusCode.startsWith('5')) return '#f44336'; // Red for 5xx
+ return '#9e9e9e'; // Gray for others
+ }
+
+ // Helper function to calculate quality score
+ function calculateQualityScore() {
+ const weights = {
+ errorStatusCoverage: 0.6, // 60% weight for error status coverage
+ positiveNegativeRatio: 0.4 // 40% weight for positive/negative ratio
+ };
+
+ const errorStatusScore = negativeMetrics.negative.coverage;
+
+ // Calculate positive/negative ratio score (ideal ratio is around 70/30)
+ const totalTests = negativeMetrics.positive.total + negativeMetrics.negative.total;
+ const negativeRatio = totalTests > 0 ? (negativeMetrics.negative.total / totalTests) * 100 : 0;
+ const idealRatio = 30; // 30% negative tests is ideal
+ const ratioScore = Math.max(0, 100 - Math.abs(negativeRatio - idealRatio) * 2);
+
+ return (errorStatusScore * weights.errorStatusCoverage) +
+ (ratioScore * weights.positiveNegativeRatio);
+ }
+
+ // Helper function to get score color
+ function getScoreColor(score) {
+ if (score >= 80) return '#4caf50'; // Green
+ if (score >= 60) return '#ff9800'; // Orange
+ if (score >= 40) return '#ffeb3b'; // Yellow
+ return '#f44336'; // Red
+ }
+
+ // Helper function to get score assessment
+ function getScoreAssessment(score) {
+ if (score >= 80) return '๐ Excellent';
+ if (score >= 60) return 'โ
Good';
+ if (score >= 40) return 'โ ๏ธ Fair';
+ return 'โ Poor';
+ }
+
+ // Helper function to generate testing recommendations
+ function generateTestingRecommendations() {
+ const recommendations = [];
+
+ // Check for missing authentication/authorization tests
+ const authOps = coverageData.filter(item =>
+ item.statusCode === '401' || item.statusCode === '403'
+ );
+ const unmatchedAuthOps = authOps.filter(item => item.unmatched);
+ if (unmatchedAuthOps.length > 0) {
+ recommendations.push('Add authentication/authorization negative tests');
+ }
+
+ // Check for missing validation error tests
+ const validationOps = coverageData.filter(item =>
+ item.statusCode === '400' || item.statusCode === '422'
+ );
+ const unmatchedValidationOps = validationOps.filter(item => item.unmatched);
+ if (unmatchedValidationOps.length > 0) {
+ recommendations.push('Add input validation negative tests');
+ }
+
+ // Check for missing resource not found tests
+ const notFoundOps = coverageData.filter(item => item.statusCode === '404');
+ const unmatchedNotFoundOps = notFoundOps.filter(item => item.unmatched);
+ if (unmatchedNotFoundOps.length > 0) {
+ recommendations.push('Add resource not found tests');
+ }
+
+ // Check for missing conflict tests
+ const conflictOps = coverageData.filter(item => item.statusCode === '409');
+ const unmatchedConflictOps = conflictOps.filter(item => item.unmatched);
+ if (unmatchedConflictOps.length > 0) {
+ recommendations.push('Add resource conflict tests');
+ }
+
+ // Check negative test ratio
+ const totalTests = negativeMetrics.positive.total + negativeMetrics.negative.total;
+ const negativeRatio = totalTests > 0 ? (negativeMetrics.negative.total / totalTests) * 100 : 0;
+ if (negativeRatio < 20) {
+ recommendations.push('Increase negative test coverage (recommended: 20-30%)');
+ }
+
+ return recommendations;
+ }
+
// Export PDF via window.print() approach
function exportToPDF() {
window.print();
@@ -1011,4 +1390,4 @@ function generateHtmlReport({ coverage, coverageItems, meta }) {
}
// Export the function
-module.exports = { generateHtmlReport };
+module.exports = { generateHtmlReport, calculateNegativeTestingMetrics };
diff --git a/test/fixtures/strict-validation-api.yaml b/test/fixtures/strict-validation-api.yaml
index 55fabf4..26d4c91 100644
--- a/test/fixtures/strict-validation-api.yaml
+++ b/test/fixtures/strict-validation-api.yaml
@@ -35,6 +35,14 @@ paths:
responses:
'200':
description: Users retrieved successfully
+ '400':
+ description: Bad request - invalid query parameters
+ '401':
+ description: Unauthorized - authentication required
+ '403':
+ description: Forbidden - insufficient permissions
+ '429':
+ description: Too many requests - rate limit exceeded
post:
operationId: createUserWithJsonBody
@@ -61,6 +69,12 @@ paths:
responses:
'201':
description: User created successfully
+ '400':
+ description: Bad request - invalid user data
+ '409':
+ description: Conflict - user already exists
+ '422':
+ description: Unprocessable entity - validation errors
/products/{id}:
parameters:
@@ -101,6 +115,12 @@ paths:
responses:
'200':
description: Product updated successfully
+ '400':
+ description: Bad request - invalid product data
+ '404':
+ description: Product not found
+ '409':
+ description: Conflict - product version mismatch
/orders:
post:
@@ -120,6 +140,10 @@ paths:
responses:
'201':
description: Order created successfully
+ '400':
+ description: Bad request - invalid order data
+ '422':
+ description: Unprocessable entity - invalid form data
/search:
get:
@@ -152,4 +176,8 @@ paths:
minimum: 0
responses:
'200':
- description: Search results
\ No newline at end of file
+ description: Search results
+ '400':
+ description: Bad request - invalid search parameters
+ '404':
+ description: No results found
\ No newline at end of file
diff --git a/test/fixtures/strict-validation-collection.json b/test/fixtures/strict-validation-collection.json
index b03eb1c..2809086 100644
--- a/test/fixtures/strict-validation-collection.json
+++ b/test/fixtures/strict-validation-collection.json
@@ -292,6 +292,823 @@
]
}
]
+ },
+ {
+ "name": "Negative Testing - Error Status Codes",
+ "item": [
+ {
+ "name": "Get Users - Bad Request (400)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/users?status=active&limit=-1",
+ "host": ["api", "example", "com"],
+ "path": ["users"],
+ "query": [
+ {"key": "status", "value": "active"},
+ {"key": "limit", "value": "-1"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 400', function () {",
+ " pm.response.to.have.status(400);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Get Users - Unauthorized (401)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/users?status=active&limit=10",
+ "host": ["api", "example", "com"],
+ "path": ["users"],
+ "query": [
+ {"key": "status", "value": "active"},
+ {"key": "limit", "value": "10"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 401', function () {",
+ " pm.response.to.have.status(401);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Get Users - Forbidden (403)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/users?status=active&limit=10",
+ "host": ["api", "example", "com"],
+ "path": ["users"],
+ "query": [
+ {"key": "status", "value": "active"},
+ {"key": "limit", "value": "10"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 403', function () {",
+ " pm.response.to.have.status(403);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Get Users - Rate Limited (429)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/users?status=active&limit=10",
+ "host": ["api", "example", "com"],
+ "path": ["users"],
+ "query": [
+ {"key": "status", "value": "active"},
+ {"key": "limit", "value": "10"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 429', function () {",
+ " pm.response.to.have.status(429);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Create User - Bad Request (400)",
+ "request": {
+ "method": "POST",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"email\": \"invalid-email\",\n \"name\": \"A\",\n \"age\": 10\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/users",
+ "host": ["api", "example", "com"],
+ "path": ["users"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 400', function () {",
+ " pm.response.to.have.status(400);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Create User - Conflict (409)",
+ "request": {
+ "method": "POST",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"email\": \"existing@example.com\",\n \"name\": \"John Doe\",\n \"age\": 25\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/users",
+ "host": ["api", "example", "com"],
+ "path": ["users"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 409', function () {",
+ " pm.response.to.have.status(409);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Create User - Validation Error (422)",
+ "request": {
+ "method": "POST",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"email\": \"test@example.com\"\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/users",
+ "host": ["api", "example", "com"],
+ "path": ["users"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 422', function () {",
+ " pm.response.to.have.status(422);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Update Product - Not Found (404)",
+ "request": {
+ "method": "PATCH",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"name\": \"Updated Product\",\n \"price\": 29.99\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/products/999999?validate=true",
+ "host": ["api", "example", "com"],
+ "path": ["products", "999999"],
+ "query": [
+ {"key": "validate", "value": "true"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 404', function () {",
+ " pm.response.to.have.status(404);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Update Product - Version Conflict (409)",
+ "request": {
+ "method": "PATCH",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"name\": \"Updated Product\",\n \"price\": -10\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/products/123",
+ "host": ["api", "example", "com"],
+ "path": ["products", "123"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 409', function () {",
+ " pm.response.to.have.status(409);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Create Order - Invalid Form Data (422)",
+ "request": {
+ "method": "POST",
+ "header": [],
+ "body": {
+ "mode": "urlencoded",
+ "urlencoded": [
+ {"key": "product_id", "value": "invalid"},
+ {"key": "quantity", "value": "-1"}
+ ]
+ },
+ "url": {
+ "raw": "https://api.example.com/orders",
+ "host": ["api", "example", "com"],
+ "path": ["orders"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 422', function () {",
+ " pm.response.to.have.status(422);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Search - Invalid Parameters (400)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/search?q=ab&price_min=-100",
+ "host": ["api", "example", "com"],
+ "path": ["search"],
+ "query": [
+ {"key": "q", "value": "ab"},
+ {"key": "price_min", "value": "-100"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 400', function () {",
+ " pm.response.to.have.status(400);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Search - No Results (404)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/search?q=nonexistent_product_xyz123",
+ "host": ["api", "example", "com"],
+ "path": ["search"],
+ "query": [
+ {"key": "q", "value": "nonexistent_product_xyz123"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 404', function () {",
+ " pm.response.to.have.status(404);",
+ "});"
+ ]
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "Boundary Value Testing",
+ "item": [
+ {
+ "name": "Get Users - Limit Boundary Max (100)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/users?status=active&limit=100",
+ "host": ["api", "example", "com"],
+ "path": ["users"],
+ "query": [
+ {"key": "status", "value": "active"},
+ {"key": "limit", "value": "100"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 200', function () {",
+ " pm.response.to.have.status(200);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Get Users - Limit Boundary Min (1)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/users?status=active&limit=1",
+ "host": ["api", "example", "com"],
+ "path": ["users"],
+ "query": [
+ {"key": "status", "value": "active"},
+ {"key": "limit", "value": "1"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 200', function () {",
+ " pm.response.to.have.status(200);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Get Users - Limit Above Max (400)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/users?status=active&limit=101",
+ "host": ["api", "example", "com"],
+ "path": ["users"],
+ "query": [
+ {"key": "status", "value": "active"},
+ {"key": "limit", "value": "101"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 400', function () {",
+ " pm.response.to.have.status(400);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Get Users - Limit Below Min (400)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/users?status=active&limit=0",
+ "host": ["api", "example", "com"],
+ "path": ["users"],
+ "query": [
+ {"key": "status", "value": "active"},
+ {"key": "limit", "value": "0"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 400', function () {",
+ " pm.response.to.have.status(400);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Create User - Age Boundary Min (18)",
+ "request": {
+ "method": "POST",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"email\": \"boundary@example.com\",\n \"name\": \"Boundary Test\",\n \"age\": 18\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/users",
+ "host": ["api", "example", "com"],
+ "path": ["users"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 201', function () {",
+ " pm.response.to.have.status(201);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Create User - Age Below Min (400)",
+ "request": {
+ "method": "POST",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"email\": \"underage@example.com\",\n \"name\": \"Under Age\",\n \"age\": 17\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/users",
+ "host": ["api", "example", "com"],
+ "path": ["users"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 400', function () {",
+ " pm.response.to.have.status(400);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Create User - Name Length Min (2)",
+ "request": {
+ "method": "POST",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"email\": \"min@example.com\",\n \"name\": \"AB\",\n \"age\": 25\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/users",
+ "host": ["api", "example", "com"],
+ "path": ["users"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 201', function () {",
+ " pm.response.to.have.status(201);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Create User - Name Too Short (400)",
+ "request": {
+ "method": "POST",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"email\": \"short@example.com\",\n \"name\": \"A\",\n \"age\": 25\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/users",
+ "host": ["api", "example", "com"],
+ "path": ["users"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 400', function () {",
+ " pm.response.to.have.status(400);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Update Product - Price Boundary Zero (0)",
+ "request": {
+ "method": "PATCH",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"name\": \"Free Product\",\n \"price\": 0\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/products/123",
+ "host": ["api", "example", "com"],
+ "path": ["products", "123"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 200', function () {",
+ " pm.response.to.have.status(200);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Update Product - Negative Price (400)",
+ "request": {
+ "method": "PATCH",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"name\": \"Invalid Product\",\n \"price\": -1\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/products/123",
+ "host": ["api", "example", "com"],
+ "path": ["products", "123"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 400', function () {",
+ " pm.response.to.have.status(400);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Search - Query Length Min (3)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/search?q=abc",
+ "host": ["api", "example", "com"],
+ "path": ["search"],
+ "query": [
+ {"key": "q", "value": "abc"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 200', function () {",
+ " pm.response.to.have.status(200);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Search - Query Too Short (400)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/search?q=ab",
+ "host": ["api", "example", "com"],
+ "path": ["search"],
+ "query": [
+ {"key": "q", "value": "ab"}
+ ]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 400', function () {",
+ " pm.response.to.have.status(400);",
+ "});"
+ ]
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "Unsupported Operations Testing",
+ "item": [
+ {
+ "name": "Delete User - Method Not Allowed (405)",
+ "request": {
+ "method": "DELETE",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/users/123",
+ "host": ["api", "example", "com"],
+ "path": ["users", "123"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 405', function () {",
+ " pm.response.to.have.status(405);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Put User - Method Not Allowed (405)",
+ "request": {
+ "method": "PUT",
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\n \"email\": \"put@example.com\",\n \"name\": \"Put User\"\n}"
+ },
+ "url": {
+ "raw": "https://api.example.com/users/123",
+ "host": ["api", "example", "com"],
+ "path": ["users", "123"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 405', function () {",
+ " pm.response.to.have.status(405);",
+ "});"
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "Invalid Endpoint - Not Found (404)",
+ "request": {
+ "method": "GET",
+ "header": [],
+ "url": {
+ "raw": "https://api.example.com/nonexistent",
+ "host": ["api", "example", "com"],
+ "path": ["nonexistent"]
+ }
+ },
+ "event": [
+ {
+ "listen": "test",
+ "script": {
+ "type": "text/javascript",
+ "exec": [
+ "pm.test('Status is 404', function () {",
+ " pm.response.to.have.status(404);",
+ "});"
+ ]
+ }
+ }
+ ]
+ }
+ ]
}
]
}
\ No newline at end of file
diff --git a/test/fixtures/strict-validation-newman.json b/test/fixtures/strict-validation-newman.json
index 3645e04..6fb4450 100644
--- a/test/fixtures/strict-validation-newman.json
+++ b/test/fixtures/strict-validation-newman.json
@@ -278,6 +278,175 @@
"error": null
}
]
+ },
+ {
+ "id": "exec-9",
+ "item": {
+ "name": "Get Users - Bad Request (400)",
+ "id": "item-9"
+ },
+ "request": {
+ "method": "GET",
+ "url": {
+ "raw": "https://api.example.com/users?status=active&limit=-1",
+ "host": ["api", "example", "com"],
+ "path": ["users"],
+ "query": [
+ {"key": "status", "value": "active"},
+ {"key": "limit", "value": "-1"}
+ ]
+ },
+ "header": []
+ },
+ "response": {
+ "status": "Bad Request",
+ "code": 400,
+ "responseTime": 120,
+ "responseSize": 256
+ },
+ "assertions": [
+ {
+ "assertion": "Status is 400",
+ "error": null
+ }
+ ]
+ },
+ {
+ "id": "exec-10",
+ "item": {
+ "name": "Get Users - Unauthorized (401)",
+ "id": "item-10"
+ },
+ "request": {
+ "method": "GET",
+ "url": {
+ "raw": "https://api.example.com/users?status=active&limit=10",
+ "host": ["api", "example", "com"],
+ "path": ["users"],
+ "query": [
+ {"key": "status", "value": "active"},
+ {"key": "limit", "value": "10"}
+ ]
+ },
+ "header": []
+ },
+ "response": {
+ "status": "Unauthorized",
+ "code": 401,
+ "responseTime": 95,
+ "responseSize": 128
+ },
+ "assertions": [
+ {
+ "assertion": "Status is 401",
+ "error": null
+ }
+ ]
+ },
+ {
+ "id": "exec-11",
+ "item": {
+ "name": "Create User - Conflict (409)",
+ "id": "item-11"
+ },
+ "request": {
+ "method": "POST",
+ "url": {
+ "raw": "https://api.example.com/users",
+ "host": ["api", "example", "com"],
+ "path": ["users"]
+ },
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\"email\": \"existing@example.com\", \"name\": \"John Doe\", \"age\": 25}"
+ }
+ },
+ "response": {
+ "status": "Conflict",
+ "code": 409,
+ "responseTime": 140,
+ "responseSize": 200
+ },
+ "assertions": [
+ {
+ "assertion": "Status is 409",
+ "error": null
+ }
+ ]
+ },
+ {
+ "id": "exec-12",
+ "item": {
+ "name": "Update Product - Not Found (404)",
+ "id": "item-12"
+ },
+ "request": {
+ "method": "PATCH",
+ "url": {
+ "raw": "https://api.example.com/products/999999?validate=true",
+ "host": ["api", "example", "com"],
+ "path": ["products", "999999"],
+ "query": [
+ {"key": "validate", "value": "true"}
+ ]
+ },
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\"name\": \"Updated Product\", \"price\": 29.99}"
+ }
+ },
+ "response": {
+ "status": "Not Found",
+ "code": 404,
+ "responseTime": 110,
+ "responseSize": 150
+ },
+ "assertions": [
+ {
+ "assertion": "Status is 404",
+ "error": null
+ }
+ ]
+ },
+ {
+ "id": "exec-13",
+ "item": {
+ "name": "Create User - Validation Error (422)",
+ "id": "item-13"
+ },
+ "request": {
+ "method": "POST",
+ "url": {
+ "raw": "https://api.example.com/users",
+ "host": ["api", "example", "com"],
+ "path": ["users"]
+ },
+ "header": [
+ {"key": "Content-Type", "value": "application/json"}
+ ],
+ "body": {
+ "mode": "raw",
+ "raw": "{\"email\": \"test@example.com\"}"
+ }
+ },
+ "response": {
+ "status": "Unprocessable Entity",
+ "code": 422,
+ "responseTime": 125,
+ "responseSize": 300
+ },
+ "assertions": [
+ {
+ "assertion": "Status is 422",
+ "error": null
+ }
+ ]
}
]
}
diff --git a/test/negative-testing-coverage.test.js b/test/negative-testing-coverage.test.js
new file mode 100644
index 0000000..8917539
--- /dev/null
+++ b/test/negative-testing-coverage.test.js
@@ -0,0 +1,493 @@
+const { matchOperationsDetailed } = require('../lib/match');
+const { loadAndParseSpec, extractOperationsFromSpec } = require('../lib/swagger');
+const { loadPostmanCollection, extractRequestsFromPostman } = require('../lib/postman');
+const { loadNewmanReport, extractRequestsFromNewman } = require('../lib/newman');
+const fs = require('fs');
+const path = require('path');
+
+describe('Negative Testing Coverage Analysis', () => {
+ let strictApiSpec;
+ let strictSpecOperations;
+ let strictPostmanRequests;
+ let strictNewmanRequests;
+
+ beforeAll(async () => {
+ // Load the strict validation API spec
+ const specPath = path.resolve(__dirname, 'fixtures/strict-validation-api.yaml');
+ strictApiSpec = await loadAndParseSpec(specPath);
+ strictSpecOperations = extractOperationsFromSpec(strictApiSpec, false);
+
+ // Load Postman collection with negative tests
+ const collectionPath = path.resolve(__dirname, 'fixtures/strict-validation-collection.json');
+ const postmanCollection = loadPostmanCollection(collectionPath);
+ strictPostmanRequests = extractRequestsFromPostman(postmanCollection, false);
+
+ // Load Newman report
+ const newmanPath = path.resolve(__dirname, 'fixtures/strict-validation-newman.json');
+ const newmanReport = loadNewmanReport(newmanPath);
+ strictNewmanRequests = extractRequestsFromNewman(newmanReport, false);
+ });
+
+ describe('Error Status Code Coverage Analysis', () => {
+ test('should identify coverage for 4xx client error status codes', () => {
+ const coverageItems = matchOperationsDetailed(strictSpecOperations, strictPostmanRequests, {
+ verbose: false,
+ strictQuery: false,
+ strictBody: false
+ });
+
+ // Find operations that have 4xx status codes defined
+ const clientErrorOps = coverageItems.filter(item =>
+ item.statusCode && item.statusCode.startsWith('4')
+ );
+
+ expect(clientErrorOps.length).toBeGreaterThan(0);
+
+ // Check that we have test coverage for common client errors
+ const badRequestOp = coverageItems.find(item => item.statusCode === '400');
+ const unauthorizedOp = coverageItems.find(item => item.statusCode === '401');
+ const forbiddenOp = coverageItems.find(item => item.statusCode === '403');
+ const notFoundOp = coverageItems.find(item => item.statusCode === '404');
+
+ expect(badRequestOp).toBeDefined();
+ expect(unauthorizedOp).toBeDefined();
+ expect(forbiddenOp).toBeDefined();
+ expect(notFoundOp).toBeDefined();
+ });
+
+ test('should identify coverage for 5xx server error status codes', () => {
+ const coverageItems = matchOperationsDetailed(strictSpecOperations, strictPostmanRequests, {
+ verbose: false,
+ strictQuery: false,
+ strictBody: false
+ });
+
+ // Find operations that have 5xx status codes defined (if any)
+ const serverErrorOps = coverageItems.filter(item =>
+ item.statusCode && item.statusCode.startsWith('5')
+ );
+
+ // Note: Current spec doesn't define 5xx errors, this is expected
+ // This test demonstrates how to analyze server error coverage
+ expect(Array.isArray(serverErrorOps)).toBe(true);
+ });
+
+ test('should match negative test cases for error status codes', () => {
+ const coverageItems = matchOperationsDetailed(strictSpecOperations, strictPostmanRequests, {
+ verbose: false,
+ strictQuery: false,
+ strictBody: false
+ });
+
+ // Find a 400 error operation and check if it has matching negative tests
+ const badRequestOp = coverageItems.find(item => item.statusCode === '400');
+ expect(badRequestOp).toBeDefined();
+
+ if (badRequestOp && !badRequestOp.unmatched) {
+ const negativeTestMatch = badRequestOp.matchedRequests.find(req =>
+ req.name.includes('400') || req.name.includes('Bad Request')
+ );
+ expect(negativeTestMatch).toBeDefined();
+ }
+ });
+
+ test('should provide negative testing recommendations', () => {
+ const coverageItems = matchOperationsDetailed(strictSpecOperations, strictPostmanRequests, {
+ verbose: false,
+ strictQuery: false,
+ strictBody: false
+ });
+
+ // Analyze which error status codes are defined but not tested
+ const errorStatusCodes = ['400', '401', '403', '404', '409', '422', '429'];
+ const definedErrorOps = coverageItems.filter(item =>
+ errorStatusCodes.includes(item.statusCode)
+ );
+
+ const unmatchedErrorOps = definedErrorOps.filter(item => item.unmatched);
+
+ // We should have some error operations defined
+ expect(definedErrorOps.length).toBeGreaterThan(0);
+
+ // Log recommendations for missing negative tests
+ if (unmatchedErrorOps.length > 0) {
+ console.log('โ ๏ธ Missing negative test coverage for:');
+ unmatchedErrorOps.forEach(op => {
+ console.log(` - ${op.method} ${op.path} (${op.statusCode})`);
+ });
+ }
+ });
+ });
+
+ describe('Boundary Value Testing Coverage Analysis', () => {
+ test('should identify boundary value test coverage for numeric parameters', () => {
+ const coverageItems = matchOperationsDetailed(strictSpecOperations, strictPostmanRequests, {
+ verbose: false,
+ strictQuery: false,
+ strictBody: false
+ });
+
+ // Look for operations that test boundary values
+ const boundaryTests = strictPostmanRequests.filter(req =>
+ req.name.includes('Boundary') ||
+ req.name.includes('Min') ||
+ req.name.includes('Max') ||
+ req.name.includes('Above') ||
+ req.name.includes('Below')
+ );
+
+ expect(boundaryTests.length).toBeGreaterThan(0);
+
+ // Check for specific boundary test patterns
+ const minBoundaryTest = boundaryTests.find(req => req.name.includes('Min'));
+ const maxBoundaryTest = boundaryTests.find(req => req.name.includes('Max'));
+ const aboveBoundaryTest = boundaryTests.find(req => req.name.includes('Above'));
+ const belowBoundaryTest = boundaryTests.find(req => req.name.includes('Below'));
+
+ expect(minBoundaryTest).toBeDefined();
+ expect(maxBoundaryTest).toBeDefined();
+ expect(aboveBoundaryTest).toBeDefined();
+ expect(belowBoundaryTest).toBeDefined();
+ });
+
+ test('should analyze string length boundary testing', () => {
+ const boundaryTests = strictPostmanRequests.filter(req =>
+ req.name.includes('Length') ||
+ req.name.includes('Short') ||
+ req.name.includes('Long')
+ );
+
+ expect(boundaryTests.length).toBeGreaterThan(0);
+ });
+
+ test('should identify missing boundary value tests', () => {
+ // Analyze which parameters have constraints but no boundary tests
+ const opsWithConstraints = strictSpecOperations.filter(op => {
+ if (!op.parameters) return false;
+ return op.parameters.some(param =>
+ param.schema && (
+ param.schema.minimum !== undefined ||
+ param.schema.maximum !== undefined ||
+ param.schema.minLength !== undefined ||
+ param.schema.maxLength !== undefined
+ )
+ );
+ });
+
+ expect(opsWithConstraints.length).toBeGreaterThan(0);
+ console.log(`๐ Found ${opsWithConstraints.length} operations with parameter constraints`);
+ });
+ });
+
+ describe('Invalid Input Testing Coverage Analysis', () => {
+ test('should identify invalid data type testing', () => {
+ const invalidTests = strictPostmanRequests.filter(req =>
+ req.name.includes('Invalid') ||
+ req.name.includes('Wrong') ||
+ req.name.includes('Bad')
+ );
+
+ expect(invalidTests.length).toBeGreaterThan(0);
+
+ // Check for specific invalid input patterns
+ const invalidEnumTest = invalidTests.find(req => req.name.includes('Enum'));
+ const invalidJsonTest = invalidTests.find(req => req.name.includes('JSON'));
+ const invalidPatternTest = invalidTests.find(req => req.name.includes('Pattern'));
+
+ expect(invalidEnumTest).toBeDefined();
+ expect(invalidJsonTest).toBeDefined();
+ expect(invalidPatternTest).toBeDefined();
+ });
+
+ test('should analyze content type mismatch testing', () => {
+ const contentTypeMismatchTests = strictPostmanRequests.filter(req =>
+ (req.name.includes('Form Data') && req.name.includes('JSON')) ||
+ req.name.includes('Instead of')
+ );
+
+ expect(contentTypeMismatchTests.length).toBeGreaterThan(0);
+ });
+
+ test('should identify malformed request testing', () => {
+ const malformedTests = strictPostmanRequests.filter(req =>
+ req.bodyInfo &&
+ req.bodyInfo.mode === 'raw' &&
+ req.bodyInfo.content &&
+ req.bodyInfo.content.includes('invalid')
+ );
+
+ expect(malformedTests.length).toBeGreaterThan(0);
+ });
+ });
+
+ describe('Unsupported Operations Testing Coverage Analysis', () => {
+ test('should identify unsupported HTTP method testing', () => {
+ const unsupportedMethodTests = strictPostmanRequests.filter(req =>
+ req.name.includes('Method Not Allowed') ||
+ req.name.includes('405')
+ );
+
+ expect(unsupportedMethodTests.length).toBeGreaterThan(0);
+ });
+
+ test('should identify invalid endpoint testing', () => {
+ const invalidEndpointTests = strictPostmanRequests.filter(req =>
+ req.name.includes('Invalid Endpoint') ||
+ req.rawUrl.includes('nonexistent')
+ );
+
+ expect(invalidEndpointTests.length).toBeGreaterThan(0);
+ });
+
+ test('should analyze coverage for unsupported operations', () => {
+ // Test requests for methods not defined in the spec
+ const specMethods = strictSpecOperations.map(op => op.method.toUpperCase());
+ const testMethods = strictPostmanRequests.map(req => req.method.toUpperCase());
+
+ const unsupportedMethods = testMethods.filter(method =>
+ !specMethods.includes(method)
+ );
+
+ // We should have some tests for unsupported methods (like DELETE, PUT on /users)
+ expect(unsupportedMethods.length).toBeGreaterThan(0);
+ console.log(`๐ซ Testing ${unsupportedMethods.length} unsupported method(s): ${[...new Set(unsupportedMethods)].join(', ')}`);
+ });
+ });
+
+ describe('Negative Testing Coverage Metrics', () => {
+ test('should calculate overall negative testing coverage percentage', () => {
+ const coverageItems = matchOperationsDetailed(strictSpecOperations, strictPostmanRequests, {
+ verbose: false,
+ strictQuery: false,
+ strictBody: false
+ });
+
+ // Calculate positive vs negative test coverage
+ const positiveStatusCodes = ['200', '201', '202', '204'];
+ const negativeStatusCodes = ['400', '401', '403', '404', '409', '422', '429', '500', '502', '503'];
+
+ const positiveOps = coverageItems.filter(item =>
+ positiveStatusCodes.includes(item.statusCode)
+ );
+ const negativeOps = coverageItems.filter(item =>
+ negativeStatusCodes.includes(item.statusCode)
+ );
+
+ const positiveMatched = positiveOps.filter(item => !item.unmatched).length;
+ const negativeMatched = negativeOps.filter(item => !item.unmatched).length;
+
+ const positiveCoverage = positiveOps.length > 0 ? (positiveMatched / positiveOps.length) * 100 : 0;
+ const negativeCoverage = negativeOps.length > 0 ? (negativeMatched / negativeOps.length) * 100 : 0;
+
+ console.log(`โ
Positive test coverage: ${positiveCoverage.toFixed(1)}% (${positiveMatched}/${positiveOps.length})`);
+ console.log(`โ Negative test coverage: ${negativeCoverage.toFixed(1)}% (${negativeMatched}/${negativeOps.length})`);
+
+ expect(positiveCoverage).toBeGreaterThan(0);
+ expect(negativeCoverage).toBeGreaterThan(0);
+ });
+
+ test('should provide negative testing recommendations based on QA best practices', () => {
+ const coverageItems = matchOperationsDetailed(strictSpecOperations, strictPostmanRequests, {
+ verbose: false,
+ strictQuery: false,
+ strictBody: false
+ });
+
+ const recommendations = [];
+
+ // Check for missing authentication/authorization tests
+ const authOps = coverageItems.filter(item =>
+ item.statusCode === '401' || item.statusCode === '403'
+ );
+ const unmatchedAuthOps = authOps.filter(item => item.unmatched);
+ if (unmatchedAuthOps.length > 0) {
+ recommendations.push('Add authentication/authorization negative tests');
+ }
+
+ // Check for missing validation error tests
+ const validationOps = coverageItems.filter(item =>
+ item.statusCode === '400' || item.statusCode === '422'
+ );
+ const unmatchedValidationOps = validationOps.filter(item => item.unmatched);
+ if (unmatchedValidationOps.length > 0) {
+ recommendations.push('Add input validation negative tests');
+ }
+
+ // Check for missing resource not found tests
+ const notFoundOps = coverageItems.filter(item => item.statusCode === '404');
+ const unmatchedNotFoundOps = notFoundOps.filter(item => item.unmatched);
+ if (unmatchedNotFoundOps.length > 0) {
+ recommendations.push('Add resource not found tests');
+ }
+
+ // Check for missing conflict tests
+ const conflictOps = coverageItems.filter(item => item.statusCode === '409');
+ const unmatchedConflictOps = conflictOps.filter(item => item.unmatched);
+ if (unmatchedConflictOps.length > 0) {
+ recommendations.push('Add resource conflict tests');
+ }
+
+ console.log('๐ฏ Negative Testing Recommendations:');
+ if (recommendations.length === 0) {
+ console.log(' โ
Good negative test coverage detected!');
+ } else {
+ recommendations.forEach(rec => console.log(` - ${rec}`));
+ }
+
+ // Always expect some form of analysis result
+ expect(Array.isArray(recommendations)).toBe(true);
+ });
+
+ test('should identify negative testing gaps in API operations', () => {
+ // Group operations by endpoint to identify gaps
+ const endpointGroups = {};
+
+ strictSpecOperations.forEach(op => {
+ const key = `${op.method.toUpperCase()} ${op.path}`;
+ if (!endpointGroups[key]) {
+ endpointGroups[key] = {
+ operation: op,
+ statusCodes: []
+ };
+ }
+ endpointGroups[key].statusCodes.push(op.statusCode);
+ });
+
+ // Analyze each endpoint for negative testing completeness
+ Object.entries(endpointGroups).forEach(([endpoint, data]) => {
+ const hasPositiveTest = data.statusCodes.some(code => ['200', '201', '202', '204'].includes(code));
+ const hasNegativeTest = data.statusCodes.some(code => ['400', '401', '403', '404', '409', '422', '429'].includes(code));
+
+ if (hasPositiveTest && !hasNegativeTest) {
+ console.log(`โ ๏ธ Missing negative tests for: ${endpoint}`);
+ }
+ });
+
+ expect(Object.keys(endpointGroups).length).toBeGreaterThan(0);
+ });
+ });
+
+ describe('Advanced Negative Testing Patterns', () => {
+ test('should analyze SQL injection and XSS prevention testing', () => {
+ // Look for test requests that include potential malicious payloads
+ const securityTests = strictPostmanRequests.filter(req => {
+ const bodyContent = req.bodyInfo?.content || '';
+ const queryParams = req.queryParams || [];
+
+ // Check for common injection patterns in test data
+ const maliciousPatterns = [
+ 'script>', '
{
+ const rateLimitTests = strictPostmanRequests.filter(req =>
+ req.name.includes('Rate Limited') ||
+ req.name.includes('429') ||
+ req.name.includes('Too Many')
+ );
+
+ expect(rateLimitTests.length).toBeGreaterThan(0);
+ console.log(`โฑ๏ธ Rate limiting tests found: ${rateLimitTests.length}`);
+ });
+
+ test('should analyze concurrency and race condition testing', () => {
+ // Look for tests that might involve concurrent operations
+ const concurrencyTests = strictPostmanRequests.filter(req =>
+ req.name.includes('Conflict') ||
+ req.name.includes('Version') ||
+ req.name.includes('409')
+ );
+
+ expect(concurrencyTests.length).toBeGreaterThan(0);
+ console.log(`๐ Concurrency/conflict tests found: ${concurrencyTests.length}`);
+ });
+
+ test('should provide comprehensive negative testing quality score', () => {
+ const coverageItems = matchOperationsDetailed(strictSpecOperations, strictPostmanRequests, {
+ verbose: false,
+ strictQuery: false,
+ strictBody: false
+ });
+
+ // Calculate a comprehensive negative testing quality score
+ const metrics = {
+ errorStatusCoverage: 0,
+ boundaryValueTests: 0,
+ invalidInputTests: 0,
+ securityTests: 0,
+ unsupportedOpTests: 0
+ };
+
+ // Error status coverage
+ const errorOps = coverageItems.filter(item =>
+ item.statusCode && ['400', '401', '403', '404', '409', '422', '429'].includes(item.statusCode)
+ );
+ const matchedErrorOps = errorOps.filter(item => !item.unmatched);
+ metrics.errorStatusCoverage = errorOps.length > 0 ? (matchedErrorOps.length / errorOps.length) * 100 : 0;
+
+ // Boundary value tests
+ const boundaryTests = strictPostmanRequests.filter(req =>
+ req.name.includes('Boundary') || req.name.includes('Min') || req.name.includes('Max')
+ );
+ metrics.boundaryValueTests = boundaryTests.length;
+
+ // Invalid input tests
+ const invalidTests = strictPostmanRequests.filter(req =>
+ req.name.includes('Invalid') || req.name.includes('Bad') || req.name.includes('Wrong')
+ );
+ metrics.invalidInputTests = invalidTests.length;
+
+ // Unsupported operation tests
+ const unsupportedTests = strictPostmanRequests.filter(req =>
+ req.name.includes('Method Not Allowed') || req.name.includes('Invalid Endpoint')
+ );
+ metrics.unsupportedOpTests = unsupportedTests.length;
+
+ // Calculate overall quality score (0-100)
+ const weights = {
+ errorStatusCoverage: 0.4, // 40% weight
+ boundaryValueTests: 0.2, // 20% weight
+ invalidInputTests: 0.2, // 20% weight
+ unsupportedOpTests: 0.2 // 20% weight
+ };
+
+ const qualityScore =
+ (metrics.errorStatusCoverage * weights.errorStatusCoverage) +
+ (Math.min(metrics.boundaryValueTests * 10, 100) * weights.boundaryValueTests) +
+ (Math.min(metrics.invalidInputTests * 10, 100) * weights.invalidInputTests) +
+ (Math.min(metrics.unsupportedOpTests * 20, 100) * weights.unsupportedOpTests);
+
+ console.log('\n๐ Negative Testing Quality Metrics:');
+ console.log(` Error Status Coverage: ${metrics.errorStatusCoverage.toFixed(1)}%`);
+ console.log(` Boundary Value Tests: ${metrics.boundaryValueTests}`);
+ console.log(` Invalid Input Tests: ${metrics.invalidInputTests}`);
+ console.log(` Unsupported Op Tests: ${metrics.unsupportedOpTests}`);
+ console.log(` Overall Quality Score: ${qualityScore.toFixed(1)}/100`);
+
+ // Provide quality assessment
+ let assessment = '';
+ if (qualityScore >= 80) assessment = '๐ Excellent';
+ else if (qualityScore >= 60) assessment = 'โ
Good';
+ else if (qualityScore >= 40) assessment = 'โ ๏ธ Fair';
+ else assessment = 'โ Poor';
+
+ console.log(` Quality Assessment: ${assessment}`);
+
+ expect(qualityScore).toBeGreaterThan(0);
+ expect(qualityScore).toBeLessThanOrEqual(100);
+ });
+ });
+});
\ No newline at end of file