scanner-comparison.yml

---
name: Secret Scanner Comparison Benchmark

on:
  workflow_dispatch:
  schedule:
    # Run weekly on Sundays at 02:00 UTC
    - cron: '0 2 * * 0'

permissions:
  contents: read

jobs:
  trufflehog:
    runs-on: ubuntu-latest
    outputs:
      count: ${{ steps.count.outputs.findings }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Run TruffleHog OSS
        run: |
          # Use TruffleHog directly to capture JSON output properly
          docker run --rm -v "$(pwd):/pwd" \
            trufflesecurity/trufflehog:latest filesystem /pwd \
            --json --only-verified > trufflehog_output.json || true
        continue-on-error: true
        id: trufflehog

      - name: Count TruffleHog findings
        id: count
        run: |
          # Count findings from TruffleHog output (it outputs JSON lines)
          count=0
          if [ -f trufflehog_output.json ]; then
            count=$(cat trufflehog_output.json | \
              grep -c "\"verified\":" || echo "0")
          fi
          echo "findings=$count" >> $GITHUB_OUTPUT
          echo "TruffleHog found $count verified secrets"

  git-secrets:
    runs-on: ubuntu-latest
    outputs:
      count: ${{ steps.count.outputs.findings }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Install git-secrets
        run: |
          git clone https://github.com/awslabs/git-secrets.git
          cd git-secrets
          sudo make install

      - name: Initialize git-secrets
        run: |
          git secrets --register-aws
          git secrets --install

      - name: Run git-secrets scan
        id: scan
        run: |
          set +e
          git secrets --scan > git_secrets_output.txt 2>&1
          exit_code=$?
          echo "exit_code=$exit_code" >> $GITHUB_OUTPUT
          cat git_secrets_output.txt
        continue-on-error: true

      - name: Count git-secrets findings
        id: count
        run: |
          count=0
          if [ -f git_secrets_output.txt ]; then
            # Count lines that indicate findings (exclude headers/empty lines)
            count=$(grep -c ".*:.*:.*" git_secrets_output.txt || echo "0")
          fi
          echo "findings=$count" >> $GITHUB_OUTPUT
          echo "git-secrets found $count secrets"

  detect-secrets:
    runs-on: ubuntu-latest
    outputs:
      count: ${{ steps.count.outputs.findings }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.14'

      - name: Install detect-secrets
        run: |
          pip install detect-secrets

      - name: Run detect-secrets scan
        run: |
          detect-secrets scan --all-files > detect_secrets_output.json
        continue-on-error: true

      - name: Count detect-secrets findings
        id: count
        run: |
          count=0
          if [ -f detect_secrets_output.json ]; then
            # Count the number of potential secrets found
            count=$(jq '.results | to_entries | map(.value | length) | \
              add // 0' detect_secrets_output.json)
          fi
          echo "findings=$count" >> $GITHUB_OUTPUT
          echo "detect-secrets found $count potential secrets"

  gitleaks:
    runs-on: ubuntu-latest
    outputs:
      count: ${{ steps.count.outputs.findings }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Install gitleaks
        run: |
          wget -O gitleaks.tar.gz \
            https://github.com/gitleaks/gitleaks/releases/download/v8.18.4/gitleaks_8.18.4_linux_x64.tar.gz
          tar -xf gitleaks.tar.gz
          sudo mv gitleaks /usr/local/bin/
          gitleaks version

      - name: Run gitleaks scan
        run: |
          gitleaks detect --source . --report-format json \
            --report-path gitleaks_output.json --no-git || \
            echo "Gitleaks scan completed"
        continue-on-error: true

      - name: Count gitleaks findings
        id: count
        run: |
          count=0
          if [ -f gitleaks_output.json ]; then
            # Count findings in JSON output
            count=$(jq 'length' gitleaks_output.json 2>/dev/null || echo "0")
          fi
          echo "findings=$count" >> $GITHUB_OUTPUT
          echo "gitleaks found $count secrets"

  gittyleaks:
    runs-on: ubuntu-latest
    outputs:
      count: ${{ steps.count.outputs.findings }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.14'

      - name: Install gittyleaks
        run: |
          pip install gittyleaks

      - name: Run gittyleaks scan
        run: |
          gittyleaks --find-anything > gittyleaks_output.txt 2>&1
        continue-on-error: true

      - name: Count gittyleaks findings
        id: count
        run: |
          count=0
          if [ -f gittyleaks_output.txt ]; then
            # Count lines that contain findings (exclude header/footer lines)
            count=$(grep ":" gittyleaks_output.txt | \
              grep -v "Bot Detective" | grep -v "^---" | \
              wc -l || echo "0")
          fi
          echo "findings=$count" >> $GITHUB_OUTPUT
          echo "gittyleaks found $count secrets"

  whispers:
    runs-on: ubuntu-latest
    outputs:
      count: ${{ steps.count.outputs.findings }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.14'

      - name: Install whispers (with timeout handling)
        run: |
          # Try to install whispers with a timeout and fallback
          timeout 300 pip install whispers || echo "Failed to install whispers"
        continue-on-error: true

      - name: Run whispers scan
        run: |
          if command -v whispers >/dev/null 2>&1; then
            whispers . --output whispers_output.json --format json || \
              echo "Whispers scan failed"
          else
            echo "Whispers not available, skipping scan"
            echo "[]" > whispers_output.json
          fi
        continue-on-error: true

      - name: Count whispers findings
        id: count
        run: |
          count=0
          if [ -f whispers_output.json ]; then
            # Count findings in JSON output
            count=$(jq 'length' whispers_output.json 2>/dev/null || echo "0")
          fi
          echo "findings=$count" >> $GITHUB_OUTPUT
          echo "whispers found $count secrets"

  trufflehog3:
    runs-on: ubuntu-latest
    outputs:
      count: ${{ steps.count.outputs.findings }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.14'

      - name: Install trufflehog3
        run: |
          pip install trufflehog3

      - name: Run trufflehog3 scan
        run: |
          trufflehog3 . --format json > trufflehog3_output.json 2>&1 || \
            echo "TruffleHog3 scan completed with warnings"
        continue-on-error: true

      - name: Count trufflehog3 findings
        id: count
        run: |
          count=0
          if [ -f trufflehog3_output.json ]; then
            # Count findings - each line that starts with '{' is a finding
            count=$(grep -c '^{' trufflehog3_output.json || echo "0")
          fi
          echo "findings=$count" >> $GITHUB_OUTPUT
          echo "trufflehog3 found $count secrets"

  summary:
    needs: [trufflehog, git-secrets, gitleaks, detect-secrets, gittyleaks,
            whispers, trufflehog3]
    runs-on: ubuntu-latest
    steps:
      - name: Create Summary Report
        run: |
          echo "# Secret Scanner Comparison Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Scanner | Secrets Found |" >> $GITHUB_STEP_SUMMARY
          echo "|---------|---------------|" >> $GITHUB_STEP_SUMMARY
          echo "| TruffleHog | ${{ needs.trufflehog.outputs.count }} |" \
            >> $GITHUB_STEP_SUMMARY
          echo "| git-secrets | ${{ needs.git-secrets.outputs.count }} |" \
            >> $GITHUB_STEP_SUMMARY
          echo "| gitleaks | ${{ needs.gitleaks.outputs.count }} |" \
            >> $GITHUB_STEP_SUMMARY
          echo "| detect-secrets |" \
            "${{ needs.detect-secrets.outputs.count }} |" \
            >> $GITHUB_STEP_SUMMARY
          echo "| gittyleaks | ${{ needs.gittyleaks.outputs.count }} |" \
            >> $GITHUB_STEP_SUMMARY
          echo "| whispers | ${{ needs.whispers.outputs.count }} |" \
            >> $GITHUB_STEP_SUMMARY
          echo "| trufflehog3 | ${{ needs.trufflehog3.outputs.count }} |" \
            >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Total unique scanning tools tested:** 7" \
            >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "_This benchmark helps understand the relative effectiveness" \
            "of different secret scanning tools on the OWASP WrongSecrets" \
            "repository._" >> $GITHUB_STEP_SUMMARY

          # Also output to console
          echo "=== Secret Scanner Comparison Results ==="
          echo "TruffleHog: ${{ needs.trufflehog.outputs.count }} secrets"
          echo "git-secrets: ${{ needs.git-secrets.outputs.count }} secrets"
          echo "gitleaks: ${{ needs.gitleaks.outputs.count }} secrets"
          echo "detect-secrets: ${{ needs.detect-secrets.outputs.count }} \
            secrets"
          echo "gittyleaks: ${{ needs.gittyleaks.outputs.count }} secrets"
          echo "whispers: ${{ needs.whispers.outputs.count }} secrets"
          echo "trufflehog3: ${{ needs.trufflehog3.outputs.count }} secrets"