diff --git a/.github/skills/check-urls/README.md b/.github/skills/check-urls/README.md new file mode 100644 index 00000000..47e7a195 --- /dev/null +++ b/.github/skills/check-urls/README.md @@ -0,0 +1,28 @@ +--- +title: Check URLs Skill +description: Scan TypeScript source files for hardcoded URLs and verify they resolve +lastUpdated: 2026-03-18 +--- + +# Check URLs Skill + +A GitHub Copilot Agent Skill that finds all hardcoded `http(s)://` URLs in the TypeScript source files and verifies that each one still resolves. + +## Files in This Directory + +- **SKILL.md** — Main skill file with YAML frontmatter and detailed instructions for the agent +- **check-urls.js** — Node.js script that performs the scan and HTTP resolution checks +- **README.md** — This file + +## Quick Usage + +```bash +node .github/skills/check-urls/check-urls.js +``` + +## How to Invoke via Copilot + +Ask Copilot something like: +- "Check that all hardcoded URLs in the source code still resolve" +- "Are any of the links in the fluency hints broken?" +- "Validate all URL links in the TypeScript files" diff --git a/.github/skills/check-urls/SKILL.md b/.github/skills/check-urls/SKILL.md new file mode 100644 index 00000000..5f74be87 --- /dev/null +++ b/.github/skills/check-urls/SKILL.md @@ -0,0 +1,52 @@ +--- +name: check-urls +description: Find all hardcoded URLs in TypeScript source files and verify they resolve (return HTTP 2xx/3xx). Use when you want to validate that links in tips, hints, and documentation strings are still live. +--- + +# Check URLs Skill + +This skill scans all TypeScript source files for hardcoded `http://` and `https://` URLs and performs HTTP HEAD requests to verify each one resolves without a 4xx/5xx error. + +## When to Use This Skill + +Use this skill when you need to: +- Validate links added to fluency hints or tips in `maturityScoring.ts` +- Check that VS Code docs URLs, tech.hub.ms video links, or any other hardcoded URLs are still live +- Audit the codebase after bulk URL changes to catch 404s before a release +- Routinely health-check external references as part of a maintenance pass + +## Running the Check + +```bash +node .github/skills/check-urls/check-urls.js +``` + +The script will: +1. Recursively scan every `*.ts` file under `src/` +2. Extract all unique `https?://...` URLs (strips trailing punctuation, skips template literals) +3. Send an HTTP HEAD request to each URL (with a 10-second timeout) +4. If HEAD returns any 4xx status, automatically retry with GET — some servers (e.g. intent URLs, social sharing endpoints) return 404/405 for HEAD but correctly respond to GET +5. Print a summary showing ✅ OK, ⚠️ REDIRECT, or ❌ BROKEN for every URL +6. Exit with code `1` if any URL returns a 4xx or 5xx status on both HEAD and GET, or times out + +## Interpreting Output + +| Symbol | Meaning | +|--------|---------| +| ✅ OK | 2xx response — URL is live | +| ⚠️ REDIRECT | 3xx response — URL redirects; consider updating to the final destination | +| ❌ BROKEN | 4xx/5xx or connection failure — URL must be fixed | + +## After Finding Broken URLs + +1. **404 on tech.hub.ms**: The slug may have changed or the page was removed. Check `https://tech.hub.ms` to find the replacement and update `src/maturityScoring.ts`. +2. **404 on code.visualstudio.com**: The VS Code docs may have been reorganised. Search [VS Code docs](https://code.visualstudio.com/docs) for the relevant topic and update the link. +3. **Timeout**: May be a transient network issue. Re-run the script to confirm before changing anything. +4. After fixing, re-run `node .github/skills/check-urls/check-urls.js` to confirm all URLs resolve. +5. Run `npm run compile` to confirm the TypeScript build still passes. + +## Files in This Directory + +- **SKILL.md** — This file; instructions for the skill +- **check-urls.js** — Node.js script that performs the URL scan and resolution check +- **README.md** — Short overview of the skill diff --git a/.github/skills/check-urls/check-urls.js b/.github/skills/check-urls/check-urls.js new file mode 100644 index 00000000..68977cad --- /dev/null +++ b/.github/skills/check-urls/check-urls.js @@ -0,0 +1,203 @@ +#!/usr/bin/env node + +/** + * URL Resolution Check Script + * + * Scans all TypeScript source files under src/ for hardcoded http(s) URLs + * and verifies each one resolves (returns a non-4xx/5xx HTTP status). + * + * Usage: + * node .github/skills/check-urls/check-urls.js + * + * Exit codes: + * 0 — all URLs resolved successfully (2xx or 3xx) + * 1 — one or more URLs are broken (4xx / 5xx / timeout / connection error) + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const https = require('https'); +const http = require('http'); + +// ── Configuration ────────────────────────────────────────────────────────── + +const SRC_DIR = path.join(__dirname, '../../../src'); +const TIMEOUT_MS = 10_000; + +/** + * URL prefixes that are intentionally not real HTTP endpoints and should be + * skipped (e.g. JSON Schema meta-schemas, localhost references). + */ +const SKIP_PREFIXES = [ + 'http://json-schema.org/', + 'http://localhost', + 'https://localhost', +]; + +// ── Helpers ──────────────────────────────────────────────────────────────── + +/** Recursively collect all *.ts files under a directory. */ +function collectTsFiles(dir) { + const results = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + results.push(...collectTsFiles(full)); + } else if (entry.isFile() && entry.name.endsWith('.ts')) { + results.push(full); + } + } + return results; +} + +/** Extract all unique http(s) URLs from a string. */ +function extractUrls(text) { + // Match URLs, then strip trailing punctuation that isn't part of the URL + const raw = text.matchAll(/https?:\/\/[^\s"'`<>)\]},]+/g); + const urls = new Set(); + for (const [match] of raw) { + // Strip trailing punctuation characters that commonly appear after URLs + // in prose or markdown (e.g. "see https://example.com." or "(https://example.com)") + const url = match.replace(/[.,;:!?)>\]'"`]+$/u, ''); + // Skip template literal interpolations (e.g. https://${variable}/path) + if (url.includes('${')) { continue; } + urls.add(url); + } + return urls; +} + +/** Send an HTTP HEAD request; fall back to GET if the server returns a 4xx. + * Some servers (e.g. bsky.app intent URLs) return 404 or 405 for HEAD but + * correctly serve GET requests, so any 4xx HEAD response triggers a retry. */ +function checkUrl(urlStr) { + return checkUrlWithMethod(urlStr, 'HEAD').then(({ status, error }) => { + if (status >= 400) { + // Server may not support HEAD — retry with GET + return checkUrlWithMethod(urlStr, 'GET'); + } + return { status, error }; + }); +} + +/** Send an HTTP request with the given method and resolve with { status, error }. */ +function checkUrlWithMethod(urlStr, method) { + return new Promise((resolve) => { + let url; + try { + url = new URL(urlStr); + } catch { + resolve({ status: null, error: 'invalid URL' }); + return; + } + + const lib = url.protocol === 'https:' ? https : http; + const options = { + method, + hostname: url.hostname, + port: url.port || undefined, + path: url.pathname + url.search, + headers: { + 'User-Agent': 'copilot-token-tracker-url-checker/1.0', + }, + timeout: TIMEOUT_MS, + }; + + const req = lib.request(options, (res) => { + resolve({ status: res.statusCode }); + req.destroy(); // don't wait for body + res.resume(); + }); + + req.on('timeout', () => { + req.destroy(); + resolve({ status: null, error: 'timeout' }); + }); + + req.on('error', (err) => { + resolve({ status: null, error: err.message }); + }); + + req.end(); + }); +} + +// ── Main ─────────────────────────────────────────────────────────────────── + +async function main() { + // 1. Collect all TypeScript files + if (!fs.existsSync(SRC_DIR)) { + console.error(`❌ Source directory not found: ${SRC_DIR}`); + process.exit(1); + } + + const tsFiles = collectTsFiles(SRC_DIR); + console.log(`Scanning ${tsFiles.length} TypeScript file(s) under ${path.relative(process.cwd(), SRC_DIR)}/\n`); + + // 2. Extract all URLs, tracking which file(s) each came from + const urlSources = new Map(); // url → Set + for (const file of tsFiles) { + const content = fs.readFileSync(file, 'utf8'); + const rel = path.relative(process.cwd(), file); + for (const url of extractUrls(content)) { + if (!urlSources.has(url)) { + urlSources.set(url, new Set()); + } + urlSources.get(url).add(rel); + } + } + + // 3. Filter out known-skip prefixes + const urlsToCheck = [...urlSources.keys()].filter( + (u) => !SKIP_PREFIXES.some((prefix) => u.startsWith(prefix)) + ); + + if (urlsToCheck.length === 0) { + console.log('No URLs found to check.'); + process.exit(0); + } + + console.log(`Found ${urlsToCheck.length} unique URL(s) to check.\n`); + + // 4. Check each URL + let broken = 0; + + // Check sequentially to avoid hammering servers + for (const url of urlsToCheck.sort()) { + const sources = [...urlSources.get(url)].join(', '); + const { status, error } = await checkUrl(url); + + if (error) { + console.log(`❌ BROKEN [${error}]`); + console.log(` ${url}`); + console.log(` → ${sources}\n`); + broken++; + } else if (status >= 400) { + console.log(`❌ BROKEN [HTTP ${status}]`); + console.log(` ${url}`); + console.log(` → ${sources}\n`); + broken++; + } else if (status >= 300) { + console.log(`⚠️ REDIRECT [HTTP ${status}]`); + console.log(` ${url}`); + console.log(` → ${sources}\n`); + } else { + console.log(`✅ OK [HTTP ${status}] ${url}`); + } + } + + // 5. Summary + console.log('\n─────────────────────────────────────────'); + if (broken === 0) { + console.log(`✅ All ${urlsToCheck.length} URL(s) resolved successfully.`); + } else { + console.log(`❌ ${broken} of ${urlsToCheck.length} URL(s) are broken.`); + process.exit(1); + } +} + +main().catch((err) => { + console.error('Unexpected error:', err); + process.exit(1); +}); diff --git a/README.md b/README.md index 7ee16aa1..dd1e0715 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,8 @@ A debug-only tool that displays all fluency score rules, thresholds, and tips fo For detailed information, see [Fluency Level Viewer Documentation](docs/FLUENCY-LEVEL-VIEWER.md). +![Screenshot of the fluency score view, showing the score accross the 6 different dimensions](/docs/images/05%20Fluency%20Score.png) + ## Known Issues - The numbers shown use **actual token counts** from the LLM API when available in session logs (e.g. Copilot Chat JSONL sessions and OpenCode sessions). When actual token data is not available, the extension falls back to **estimates** computed from the text in the session logs. diff --git a/docs/FLUENCY-LEVEL-VIEWER.md b/docs/FLUENCY-LEVEL-VIEWER.md index b7c13383..e153611f 100644 --- a/docs/FLUENCY-LEVEL-VIEWER.md +++ b/docs/FLUENCY-LEVEL-VIEWER.md @@ -4,6 +4,8 @@ The Fluency Level Viewer is a debug-only tool that displays all fluency score rules, thresholds, and tips for each category and stage. This feature is designed to help developers understand how the scoring system works and what actions trigger different fluency levels. +![Screenshot of the fluency score view, showing the score accross the 6 different dimensions](/docs/images/05%20Fluency%20Score.png) + ## Availability **Important**: This feature is only available when a VS Code debugger is active. This is intentional to keep it as a development and testing tool rather than a production feature. diff --git a/src/maturityScoring.ts b/src/maturityScoring.ts index 36956742..2832196a 100644 --- a/src/maturityScoring.ts +++ b/src/maturityScoring.ts @@ -46,7 +46,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { "No slash commands or agent mode usage", ], tips: [ - "Try asking Copilot a question using the Chat panel", + "Try asking Copilot a question using the Chat panel — [▶ Chat in IDE video](https://tech.hub.ms/github-copilot/videos/chat-in-ide)", "Start with simple queries to get familiar with the interface", ], }, @@ -60,7 +60,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { "Beginning to use slash commands or agent mode", ], tips: [ - "Try [agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) for multi-file changes", + "Try [agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) for multi-file changes — [▶ Agent Mode video](https://tech.hub.ms/github-copilot/videos/agent-mode)", "Use [slash commands](https://code.visualstudio.com/docs/copilot/chat/copilot-chat#_add-context-to-your-prompts) like /explain, /fix, or /tests to give structured prompts", "Experiment with multi-turn conversations to refine responses", ], @@ -77,7 +77,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { ], tips: [ "Try [agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) for autonomous, multi-step coding tasks", - "Experiment with [different models](https://code.visualstudio.com/docs/copilot/chat/copilot-chat#_choose-a-language-model) for different tasks - use fast models for simple queries and reasoning models for complex problems", + "Experiment with [different models](https://code.visualstudio.com/docs/copilot/chat/copilot-chat#_choose-a-language-model) for different tasks - use fast models for simple queries and reasoning models for complex problems — [▶ Model selection video](https://tech.hub.ms/github-copilot/videos/model-selection)", "Explore more [slash commands](https://code.visualstudio.com/docs/copilot/chat/copilot-chat#_add-context-to-your-prompts) like /explain, /tests, or /doc to diversify your prompting", ], }, @@ -172,7 +172,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { "Not using edit mode or multi-file capabilities", ], tips: [ - "Try [agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) — it can run terminal commands, edit files, and explore your codebase autonomously", + "Try [agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) — it can run terminal commands, edit files, and explore your codebase autonomously — [▶ Agent Mode video](https://tech.hub.ms/github-copilot/videos/agent-mode)", "Start with simple tasks to see how agent mode works", ], }, @@ -186,7 +186,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { "At least 1 multi-file edit session", ], tips: [ - "Use [agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) for multi-step tasks; let it chain tools like file search, terminal, and code edits", + "Use [agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) for multi-step tasks; let it chain tools like file search, terminal, and code edits — [▶ Agent Mode video](https://tech.hub.ms/github-copilot/videos/agent-mode)", "Try edit mode for focused code changes", ], }, @@ -201,7 +201,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { ], tips: [ "Tackle complex refactoring or debugging tasks in [agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) for deeper autonomous workflows", - "Let agent mode handle multi-step tasks that span multiple files", + "Let agent mode handle multi-step tasks that span multiple files — [▶ Multi-file Edits video](https://tech.hub.ms/github-copilot/videos/multi-file-edits)", ], }, { @@ -234,7 +234,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { "No workspace agent sessions", ], tips: [ - "Try [agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) to let Copilot use built-in tools for file operations and terminal commands", + "Try [agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) to let Copilot use built-in tools for file operations and terminal commands — [▶ Agent Mode video](https://tech.hub.ms/github-copilot/videos/agent-mode)", "Explore the built-in tools available in agent mode", ], }, @@ -247,7 +247,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { "Using basic agent mode tools", ], tips: [ - "Set up [MCP servers](https://code.visualstudio.com/docs/copilot/customization/mcp-servers) to connect Copilot to external tools (databases, APIs, cloud services)", + "Set up [MCP servers](https://code.visualstudio.com/docs/copilot/customization/mcp-servers) to connect Copilot to external tools (databases, APIs, cloud services) — [▶ MCP with Azure and GitHub](https://tech.hub.ms/github-copilot/videos/mcp-with-azure-and-github)", "Explore [GitHub integrations](https://code.visualstudio.com/docs/copilot/agents/agent-tools) and advanced tools like editFiles and run_in_terminal", ], }, @@ -261,7 +261,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { "Using at least 1 MCP server", ], tips: [ - "Add more [MCP servers](https://code.visualstudio.com/docs/copilot/customization/mcp-servers) to expand Copilot's capabilities - check the VS Code MCP registry", + "Add more [MCP servers](https://code.visualstudio.com/docs/copilot/customization/mcp-servers) to expand Copilot's capabilities - check the VS Code MCP registry — [▶ MCP with Azure and GitHub](https://tech.hub.ms/github-copilot/videos/mcp-with-azure-and-github)", "Explore advanced tool combinations for complex workflows", ], }, @@ -294,7 +294,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { "Using fewer than 3 different models", ], tips: [ - "Create a [.github/copilot-instructions.md](https://code.visualstudio.com/docs/copilot/customization/custom-instructions) file with project-specific guidelines", + "Create a [.github/copilot-instructions.md](https://code.visualstudio.com/docs/copilot/customization/custom-instructions) file with project-specific guidelines — [▶ User Instructions video](https://tech.hub.ms/github-copilot/videos/user-instructions)", "Start customizing Copilot for your workflow", ], }, @@ -306,8 +306,8 @@ export function getFluencyLevelData(isDebugMode: boolean): { "At least 1 repository with custom instructions or agents.md", ], tips: [ - "Add [custom instructions](https://code.visualstudio.com/docs/copilot/customization/custom-instructions) to more repositories to standardize your Copilot experience", - "Experiment with different models for different tasks", + "Add [custom instructions](https://code.visualstudio.com/docs/copilot/customization/custom-instructions) to more repositories to standardize your Copilot experience — [▶ User Instructions video](https://tech.hub.ms/github-copilot/videos/user-instructions)", + "Experiment with [different models](https://tech.hub.ms/github-copilot/videos/model-selection) for different tasks", ], }, { @@ -320,7 +320,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { ], tips: [ "Aim for consistent customization across all projects with [instructions and agents.md](https://code.visualstudio.com/docs/copilot/customization/custom-instructions)", - "Explore 5+ models to match tasks with optimal model capabilities", + "Explore 5+ models to match tasks with optimal model capabilities — [▶ Model selection video](https://tech.hub.ms/github-copilot/videos/model-selection)", ], }, { @@ -352,7 +352,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { "Fewer than 10 explicit context references", ], tips: [ - "Use Copilot more regularly - even for quick questions", + "Use Copilot more regularly - even for quick questions — [▶ Chat in IDE video](https://tech.hub.ms/github-copilot/videos/chat-in-ide)", "Make Copilot part of your daily coding routine", ], }, @@ -365,7 +365,7 @@ export function getFluencyLevelData(isDebugMode: boolean): { "50%+ code block apply rate", ], tips: [ - "Combine [ask mode with agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) in your daily workflow", + "Combine [ask mode with agent mode](https://code.visualstudio.com/docs/copilot/agents/overview) in your daily workflow — [▶ Agent Mode video](https://tech.hub.ms/github-copilot/videos/agent-mode)", "Use explicit [context references](https://code.visualstudio.com/docs/copilot/chat/copilot-chat#_add-context-to-your-prompts) like #file, @workspace, and #selection", ], },