Paper implementation

.github/workflows/testbed-reporting.yaml

@@ -0,0 +1,292 @@
+name: Testbed Report Generation
+
+on:
+  schedule:
+    # Run every Sunday at 2 AM UTC
+    - cron: "0 2 * * 0"
+  workflow_dispatch:
+    inputs:
+      force_regenerate:
+        description: "Force regenerate all reports"
+        required: false
+        default: false
+        type: boolean
+      include_screenshots:
+        description: "Include Grafana screenshots"
+        required: false
+        default: true
+        type: boolean
+
+env:
+  PYTHON_VERSION: "3.11"
+  REPORT_OUTPUT_DIR: "testbed/reports"
+  VALIDATION_STRICT: "true"
+
+jobs:
+  generate-reports:
+    name: Generate Testbed Reports
+    runs-on: ubuntu-latest
+
+    services:
+      prometheus:
+        image: prom/prometheus:latest
+        ports:
+          - 9090:9090
+        volumes:
+          - ./testbed/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+        options: >-
+          --config.file=/etc/prometheus/prometheus.yml
+          --storage.tsdb.path=/prometheus
+          --web.console.libraries=/etc/prometheus/console_libraries
+          --web.console.templates=/etc/prometheus/consoles
+          --storage.tsdb.retention.time=200h
+          --web.enable-lifecycle
+
+      grafana:
+        image: grafana/grafana:latest
+        ports:
+          - 3000:3000
+        env:
+          GF_SECURITY_ADMIN_PASSWORD: admin
+          GF_USERS_ALLOW_SIGN_UP: false
+        volumes:
+          - ./testbed/grafana/provisioning:/etc/grafana/provisioning
+          - grafana-storage:/var/lib/grafana
+
+      ledger:
+        image: postgres:15
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_DB: testbed
+          POSTGRES_USER: testbed
+          POSTGRES_PASSWORD: testbed
+        volumes:
+          - postgres-data:/var/lib/postgresql/data
+          - ./testbed/runtime/ledger/init.sql:/docker-entrypoint-initdb.d/init.sql
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: "pip"
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            libcairo2-dev \
+            libpango1.0-dev \
+            libgdk-pixbuf2.0-dev \
+            libffi-dev \
+            shared-mime-info \
+            libpq-dev
+
+      - name: Install Python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r testbed/tools/reporter/requirements.txt
+          pip install -r requirements.txt
+
+      - name: Wait for services
+        run: |
+          echo "Waiting for services to be ready..."
+          timeout 300 bash -c 'until curl -s http://localhost:9090/-/healthy; do sleep 5; done'
+          timeout 300 bash -c 'until curl -s http://localhost:3000/api/health; do sleep 5; done'
+          timeout 300 bash -c 'until pg_isready -h localhost -p 5432 -U testbed; do sleep 5; done'
+
+      - name: Create sample data
+        run: |
+          python testbed/data/generator.py --output testbed/data/sample_data.json
+          python testbed/chaos/chaos_runner.py --config testbed/chaos/chaos_config.yaml
+
+      - name: Generate comprehensive report
+        run: |
+          python testbed/tools/reporter/generate_testbed_report.py \
+            --prometheus-url http://localhost:9090 \
+            --ledger-url http://localhost:5432 \
+            --grafana-url http://localhost:3000 \
+            --grafana-user admin \
+            --grafana-password admin \
+            --output-dir ${{ env.REPORT_OUTPUT_DIR }} \
+            --format both \
+            --include-art \
+            --include-certs \
+            --include-screenshots \
+            --validation-strict
+        env:
+          TESTBED_ID: ${{ github.run_id }}
+
+      - name: Validate report artifacts
+        run: |
+          echo "Validating report artifacts..."
+
+          # Check if reports were generated
+          if [ ! -f "${{ env.REPORT_OUTPUT_DIR }}"/*.pdf ]; then
+            echo "❌ PDF report not found"
+            exit 1
+          fi
+
+          if [ ! -f "${{ env.REPORT_OUTPUT_DIR }}"/*.html ]; then
+            echo "❌ HTML report not found"
+            exit 1
+          fi
+
+          if [ ! -f "${{ env.REPORT_OUTPUT_DIR }}"/*.json ]; then
+            echo "❌ JSON report not found"
+            exit 1
+          fi
+
+          # Validate JSON schema
+          python -c "
+          import json
+          import jsonschema
+
+          # Load schema
+          schema = {
+              'type': 'object',
+              'required': ['metadata', 'metrics', 'validation'],
+              'properties': {
+                  'metadata': {'type': 'object'},
+                  'metrics': {'type': 'object'},
+                  'validation': {'type': 'object'}
+              }
+          }
+
+          # Load and validate report
+          with open('${{ env.REPORT_OUTPUT_DIR }}' + '/' + [f for f in os.listdir('${{ env.REPORT_OUTPUT_DIR }}') if f.endswith('.json')][0], 'r') as f:
+              report = json.load(f)
+
+          jsonschema.validate(instance=report, schema=schema)
+          print('✅ JSON schema validation passed')
+          "
+
+          # Check validation results
+          python -c "
+          import json
+          import os
+
+          report_file = [f for f in os.listdir('${{ env.REPORT_OUTPUT_DIR }}') if f.endswith('.json')][0]
+          with open(os.path.join('${{ env.REPORT_OUTPUT_DIR }}', report_file), 'r') as f:
+              report = json.load(f)
+
+          validation = report.get('validation', {})
+
+          if not validation.get('artifacts_present', False):
+              print('❌ Missing artifacts detected')
+              print('Missing:', validation.get('missing_artifacts', []))
+              exit(1)
+
+          if not validation.get('schema_valid', False):
+              print('❌ Schema validation failed')
+              print('Errors:', validation.get('validation_errors', []))
+              exit(1)
+
+          print('✅ All validation checks passed')
+          "
+
+          echo "✅ Report validation completed successfully"
+
+      - name: Upload reports as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: testbed-reports-${{ github.run_number }}
+          path: ${{ env.REPORT_OUTPUT_DIR }}/*
+          retention-days: 30
+
+      - name: Upload reports to releases
+        if: github.event_name == 'workflow_dispatch'
+        uses: actions/upload-artifact@v4
+        with:
+          name: testbed-reports-release
+          path: ${{ env.REPORT_OUTPUT_DIR }}/*
+          retention-days: 90
+
+      - name: Notify on failure
+        if: failure()
+        uses: 8398a7/action-slack@v3
+        with:
+          status: failure
+          channel: "#testbed-alerts"
+          text: "Testbed report generation failed! Check the workflow for details."
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+
+  weekly-summary:
+    name: Weekly Report Summary
+    runs-on: ubuntu-latest
+    needs: generate-reports
+    if: always()
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Download reports
+        uses: actions/download-artifact@v4
+        with:
+          name: testbed-reports-${{ needs.generate-reports.outputs.run_number || github.run_number }}
+
+      - name: Generate summary
+        run: |
+          echo "📊 Weekly Testbed Report Summary" > summary.md
+          echo "Generated: $(date)" >> summary.md
+          echo "" >> summary.md
+
+          if [ -f "*.json" ]; then
+            echo "✅ Reports generated successfully" >> summary.md
+            echo "- PDF: Available" >> summary.md
+            echo "- HTML: Available" >> summary.md
+            echo "- JSON: Available" >> summary.md
+          else
+            echo "❌ Report generation failed" >> summary.md
+          fi
+
+          echo "" >> summary.md
+          echo "🔗 [View Workflow Run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> summary.md
+
+      - name: Comment on issue
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const summary = fs.readFileSync('summary.md', 'utf8');
+
+            // Find or create weekly summary issue
+            const { data: issues } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ['weekly-summary'],
+              state: 'open'
+            });
+
+            let issue;
+            if (issues.length === 0) {
+              // Create new issue
+              const { data: newIssue } = await github.rest.issues.create({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                title: `Weekly Testbed Report - ${new Date().toISOString().split('T')[0]}`,
+                body: summary,
+                labels: ['weekly-summary', 'automated']
+              });
+              issue = newIssue;
+            } else {
+              // Update existing issue
+              issue = issues[0];
+              await github.rest.issues.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issue.number,
+                body: summary
+              });
+            }
+
+            console.log(`Updated issue #${issue.number}`)