From 9db96a1b3d42115bb1222fb62c63e8c46b2127d0 Mon Sep 17 00:00:00 2001 From: G-Fourteen Date: Sun, 2 Nov 2025 01:50:34 -0700 Subject: [PATCH] Add CI workflows and stabilize image clipboard flow --- .github/workflows/main.yml | 190 +++++++++++++++++++++++++++ .github/workflows/pull-request.yml | 59 +++++++++ .gitignore | 1 + AI/app.js | 42 ++++-- README.md | 3 + package.json | 1 + playwright-server.js | 17 ++- scripts/build.mjs | 76 +++++++++++ tests/talk-to-unity.spec.ts | 204 +++++++++++++++++++++++++++-- 9 files changed, 568 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/pull-request.yml create mode 100644 scripts/build.mjs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..c8e9289 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,190 @@ +name: Main Branch Delivery + +on: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + name: Build and Upload Artifacts + runs-on: ubuntu-latest + continue-on-error: true + outputs: + result: ${{ steps.record.outputs.result }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Install dependencies + id: install_deps + run: npm ci + continue-on-error: true + + - name: Install Playwright browsers + id: install_playwright + if: ${{ steps.install_deps.outcome == 'success' }} + run: npx playwright install --with-deps + continue-on-error: true + + - name: Build static bundle + id: run_build + if: ${{ steps.install_deps.outcome == 'success' && steps.install_playwright.outcome == 'success' }} + run: npm run build + continue-on-error: true + + - name: Upload static artifact + if: ${{ steps.run_build.outcome == 'success' }} + uses: actions/upload-artifact@v4 + with: + name: static-site + path: dist + if-no-files-found: error + + - name: Upload Pages artifact + if: ${{ steps.run_build.outcome == 'success' }} + uses: actions/upload-pages-artifact@v3 + with: + path: dist + + - name: Record build status + id: record + if: always() + shell: bash + run: | + if [ "${{ steps.install_deps.outcome }}" != "success" ]; then + echo "result=install-failed" >> "$GITHUB_OUTPUT" + elif [ "${{ steps.install_playwright.outcome }}" != "success" ]; then + echo "result=playwright-failed" >> "$GITHUB_OUTPUT" + elif [ "${{ steps.run_build.outcome }}" != "success" ]; then + echo "result=build-failed" >> "$GITHUB_OUTPUT" + else + echo "result=success" >> "$GITHUB_OUTPUT" + fi + + report-build: + name: Report Build Status + runs-on: ubuntu-latest + needs: build + if: always() + steps: + - name: Publish build summary + shell: bash + run: | + { + echo "## Main Branch Build" + echo "" + echo "* Build job conclusion: **${{ needs.build.result }}**" + echo "* Build result classification: **${{ needs.build.outputs.result }}**" + echo "* Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } >> "$GITHUB_STEP_SUMMARY" + + deploy: + name: Deploy to Pages + runs-on: ubuntu-latest + needs: build + if: ${{ needs.build.outputs.result == 'success' }} + continue-on-error: true + environment: + name: github-pages + steps: + - name: Configure Pages + uses: actions/configure-pages@v5 + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + + tests: + name: Run Tests + runs-on: ubuntu-latest + needs: build + continue-on-error: true + outputs: + result: ${{ steps.test_status.outputs.result }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Install dependencies + id: tests_install_deps + run: npm ci + continue-on-error: true + + - name: Install Playwright browsers + id: tests_install_playwright + if: ${{ steps.tests_install_deps.outcome == 'success' }} + run: npx playwright install --with-deps + continue-on-error: true + + - name: Download build artifact + if: ${{ needs.build.outputs.result == 'success' && steps.tests_install_deps.outcome == 'success' && steps.tests_install_playwright.outcome == 'success' }} + uses: actions/download-artifact@v4 + with: + name: static-site + path: dist + + - name: Run Playwright suite + id: run_tests + if: ${{ needs.build.outputs.result == 'success' && steps.tests_install_deps.outcome == 'success' && steps.tests_install_playwright.outcome == 'success' }} + env: + CI: true + PLAYWRIGHT_SERVE_DIR: dist + run: npm run test:e2e + continue-on-error: true + + - name: Record test status + id: test_status + if: always() + shell: bash + run: | + if [ "${{ needs.build.outputs.result }}" != "success" ]; then + echo "result=skipped-build" >> "$GITHUB_OUTPUT" + elif [ "${{ steps.tests_install_deps.outcome }}" != "success" ]; then + echo "result=install-failed" >> "$GITHUB_OUTPUT" + elif [ "${{ steps.tests_install_playwright.outcome }}" != "success" ]; then + echo "result=playwright-failed" >> "$GITHUB_OUTPUT" + elif [ "${{ steps.run_tests.outcome }}" != "success" ]; then + echo "result=${{ steps.run_tests.outcome }}" >> "$GITHUB_OUTPUT" + else + echo "result=success" >> "$GITHUB_OUTPUT" + fi + + report-tests: + name: Report Tests Statuses + runs-on: ubuntu-latest + needs: tests + if: always() + steps: + - name: Publish test summary + shell: bash + run: | + { + echo "## Main Branch Tests" + echo "" + echo "* Test job conclusion: **${{ needs.tests.result }}**" + echo "* Test result classification: **${{ needs.tests.outputs.result }}**" + echo "* Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml new file mode 100644 index 0000000..9dc7b41 --- /dev/null +++ b/.github/workflows/pull-request.yml @@ -0,0 +1,59 @@ +name: Pull Request Quality Checks + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + workflow_dispatch: + +permissions: + contents: read + +jobs: + run-tests: + name: Run Tests + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Install Playwright browsers + run: npx playwright install --with-deps + + - name: Build static bundle + run: npm run build + + - name: Run Playwright suite + env: + CI: true + PLAYWRIGHT_SERVE_DIR: dist + run: npm run test:e2e + + report-tests: + name: Report Tests Statuses + runs-on: ubuntu-latest + needs: run-tests + if: always() + steps: + - name: Publish summary + shell: bash + run: | + { + echo "## Pull Request Test Results" + echo "" + echo "* Test job conclusion: **${{ needs.run-tests.result }}**" + echo "* Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index ae2f532..9008f8a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules/ test-results/ +dist/ diff --git a/AI/app.js b/AI/app.js index 51217f6..4f832f7 100644 --- a/AI/app.js +++ b/AI/app.js @@ -1021,7 +1021,7 @@ function parseAiDirectives(responseText) { return { cleanedText, commands: uniqueCommands }; } -async function executeAiCommand(command) { +async function executeAiCommand(command, options = {}) { if (!command) { return false; } @@ -1044,13 +1044,13 @@ async function executeAiCommand(command) { }); return true; case 'copy_image': - await copyImageToClipboard(); + await copyImageToClipboard(options.imageUrl); return true; case 'save_image': - await saveImage(); + await saveImage(options.imageUrl); return true; case 'open_image': - openImageInNewTab(); + openImageInNewTab(options.imageUrl); return true; case 'set_model_flux': currentImageModel = 'flux'; @@ -1295,13 +1295,19 @@ async function getAIResponse(userInput) { } const { cleanedText, commands } = parseAiDirectives(aiText); + const assistantMessage = cleanedText || aiText; + const imageUrlFromResponse = extractImageUrl(aiText) || extractImageUrl(assistantMessage); + const imageCommandQueue = []; for (const command of commands) { - await executeAiCommand(command); - } + const normalizedCommand = normalizeCommandValue(command); + if (['copy_image', 'save_image', 'open_image'].includes(normalizedCommand)) { + imageCommandQueue.push(normalizedCommand); + continue; + } - const assistantMessage = cleanedText || aiText; - const imageUrlFromResponse = extractImageUrl(aiText) || extractImageUrl(assistantMessage); + await executeAiCommand(normalizedCommand); + } const fallbackPrompt = buildFallbackImagePrompt(userInput, assistantMessage); let fallbackImageUrl = ''; @@ -1334,6 +1340,14 @@ async function getAIResponse(userInput) { const shouldSuppressSpeech = commands.includes('shutup') || commands.includes('stop_speaking'); + if (imageCommandQueue.length > 0) { + await heroImagePromise; + const imageTarget = selectedImageUrl || getImageUrl() || pendingHeroUrl; + for (const command of imageCommandQueue) { + await executeAiCommand(command, { imageUrl: imageTarget }); + } + } + if (!shouldSuppressSpeech) { const spokenText = sanitizeForSpeech(finalAssistantMessage); if (spokenText) { @@ -1442,8 +1456,8 @@ function updateHeroImage(imageUrl) { }); } -async function copyImageToClipboard() { - const imageUrl = getImageUrl(); +async function copyImageToClipboard(imageUrlOverride) { + const imageUrl = imageUrlOverride || getImageUrl() || pendingHeroUrl; if (!imageUrl) { return; } @@ -1459,8 +1473,8 @@ async function copyImageToClipboard() { } } -async function saveImage() { - const imageUrl = getImageUrl(); +async function saveImage(imageUrlOverride) { + const imageUrl = imageUrlOverride || getImageUrl() || pendingHeroUrl; if (!imageUrl) { return; } @@ -1484,8 +1498,8 @@ async function saveImage() { } } -function openImageInNewTab() { - const imageUrl = getImageUrl(); +function openImageInNewTab(imageUrlOverride) { + const imageUrl = imageUrlOverride || getImageUrl() || pendingHeroUrl; if (!imageUrl) { return; } diff --git a/README.md b/README.md index b77603b..221e4f9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # Talk to Unity in Plain English +![Main Branch Workflow Status](https://github.com/Unity-Lab-AI/Talk-to-Unity/actions/workflows/main.yml/badge.svg?branch=main) +![Pull Request Workflow Status](https://github.com/Unity-Lab-AI/Talk-to-Unity/actions/workflows/pull-request.yml/badge.svg) + Talk to Unity is a single web page that acts like a friendly concierge. The landing screen double-checks that your browser has everything it needs (secure connection, microphone, speech tools). Once every light turns green, a voice assistant named **Unity** wakes up so you can talk out loud and hear it answer back. ## What you need diff --git a/package.json b/package.json index f7b545a..2419602 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "1.0.0", "private": true, "scripts": { + "build": "node scripts/build.mjs", "test:e2e": "playwright test", "test:e2e:headed": "playwright test --headed", "test:e2e:ui": "playwright test --ui" diff --git a/playwright-server.js b/playwright-server.js index d382e7a..03bd2db 100644 --- a/playwright-server.js +++ b/playwright-server.js @@ -2,7 +2,20 @@ const http = require('http'); const fs = require('fs'); const path = require('path'); -const root = __dirname; +const projectRoot = __dirname; +const candidateRoot = process.env.PLAYWRIGHT_SERVE_DIR + ? path.resolve(projectRoot, process.env.PLAYWRIGHT_SERVE_DIR) + : ''; + +const hasCustomRoot = candidateRoot && fs.existsSync(candidateRoot); + +if (candidateRoot && !hasCustomRoot) { + console.warn( + `Requested Playwright serve directory "${candidateRoot}" was not found. Falling back to project root.` + ); +} + +const root = hasCustomRoot ? candidateRoot : projectRoot; const port = process.env.PORT ? Number(process.env.PORT) : 4173; const MIME_TYPES = { @@ -72,7 +85,7 @@ const server = http.createServer((req, res) => { }); server.listen(port, () => { - console.log(`Static server listening on http://127.0.0.1:${port}`); + console.log(`Static server listening on http://127.0.0.1:${port} (serving ${root})`); }); function shutdown() { diff --git a/scripts/build.mjs b/scripts/build.mjs new file mode 100644 index 0000000..ef3d8ab --- /dev/null +++ b/scripts/build.mjs @@ -0,0 +1,76 @@ +import { cp, mkdir, rm, writeFile } from 'fs/promises'; +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const projectRoot = path.resolve(__dirname, '..'); +const distDir = path.join(projectRoot, 'dist'); + +const aiSourceDir = path.join(projectRoot, 'AI'); +const excludedDirectories = new Set(['tests', 'playwright']); +const excludedExtensions = new Set(['.md', '.py']); + +async function prepareDistDirectory() { + await rm(distDir, { recursive: true, force: true }); + await mkdir(distDir, { recursive: true }); +} + +async function copyStaticEntry(relativePath) { + const source = path.join(projectRoot, relativePath); + const destination = path.join(distDir, relativePath); + await mkdir(path.dirname(destination), { recursive: true }); + await cp(source, destination, { recursive: true }); +} + +function shouldCopyAiEntry(sourcePath) { + const relative = path.relative(aiSourceDir, sourcePath); + if (!relative || relative === '') { + return true; + } + + const segments = relative.split(path.sep); + if (segments.some((segment) => excludedDirectories.has(segment))) { + return false; + } + + if (fs.statSync(sourcePath).isDirectory()) { + return true; + } + + const extension = path.extname(sourcePath).toLowerCase(); + if (excludedExtensions.has(extension)) { + return false; + } + + return true; +} + +async function copyApplicationAssets() { + const entries = ['index.html', 'landing.js', 'style.css', 'ai-instruct.txt']; + for (const entry of entries) { + await copyStaticEntry(entry); + } + + await cp(aiSourceDir, path.join(distDir, 'AI'), { + recursive: true, + filter: (source) => shouldCopyAiEntry(source) + }); +} + +async function finalizeDist() { + await writeFile(path.join(distDir, '.nojekyll'), '', 'utf8'); +} + +async function build() { + await prepareDistDirectory(); + await copyApplicationAssets(); + await finalizeDist(); + console.log('Static site ready in dist/.'); +} + +build().catch((error) => { + console.error('Static build failed:', error); + process.exitCode = 1; +}); diff --git a/tests/talk-to-unity.spec.ts b/tests/talk-to-unity.spec.ts index 2084464..98ef4c6 100644 --- a/tests/talk-to-unity.spec.ts +++ b/tests/talk-to-unity.spec.ts @@ -1,6 +1,4 @@ import { test, expect } from '@playwright/test'; -import { Buffer } from 'buffer'; - declare global { interface UnityTestHooks { getChatHistory(): Array<{ role: string; content: string }>; @@ -21,6 +19,7 @@ declare global { speechSynthesis: { speakCalls: string[]; }; + __testClipboardWrites?: unknown[]; } } @@ -141,9 +140,19 @@ test.beforeEach(async ({ page, context }) => { configurable: true }); + const clipboardWrites = []; + + Object.defineProperty(window, '__testClipboardWrites', { + value: clipboardWrites, + configurable: true, + writable: true + }); + Object.defineProperty(navigator, 'clipboard', { value: { - write() { + writes: clipboardWrites, + write(items) { + clipboardWrites.push(items); return Promise.resolve(); } }, @@ -165,16 +174,196 @@ test.beforeEach(async ({ page, context }) => { }); }); - await page.route('https://image.pollinations.ai/*', async (route) => { + await context.unroute('**/image.pollinations.ai/**').catch(() => {}); + await context.route('**/image.pollinations.ai/**', async (route) => { await route.fulfill({ status: 200, headers: { 'content-type': 'image/png', 'access-control-allow-origin': '*' }, - body: Buffer.from(MOCK_IMAGE_BASE64, 'base64') + body: MOCK_IMAGE_BASE64, + isBase64: true + }); + }); +}); + +test('landing highlights missing dependencies but allows limited launch', async ({ page }) => { + await page.addInitScript(() => { + Object.defineProperty(window, 'SpeechRecognition', { + value: undefined, + configurable: true + }); + Object.defineProperty(window, 'webkitSpeechRecognition', { + value: undefined, + configurable: true }); + Object.defineProperty(window, 'speechSynthesis', { + value: undefined, + configurable: true + }); + + Object.defineProperty(navigator, 'mediaDevices', { + value: { getUserMedia: undefined }, + configurable: true + }); + }); + + await page.goto('/index.html'); + + await page.evaluate(() => { + window.__unityLandingTestHooks?.initialize(); + window.__unityLandingTestHooks?.evaluateDependencies({ announce: true }); }); + + await page.waitForFunction(() => + document.querySelector('[data-dependency="speech-recognition"]')?.getAttribute('data-state') === 'fail' && + document.querySelector('[data-dependency="speech-synthesis"]')?.getAttribute('data-state') === 'fail' + ); + + await expect(page.locator('#dependency-summary')).toContainText(/Alerts/i); + await expect(page.locator('[data-dependency="speech-recognition"]')).toHaveAttribute('data-state', 'fail'); + await expect(page.locator('[data-dependency="speech-synthesis"]')).toHaveAttribute('data-state', 'fail'); + await expect(page.locator('[data-dependency="microphone"]')).toHaveAttribute('data-state', 'fail'); + + const launchButton = page.locator('#launch-app'); + await expect(launchButton).toBeEnabled(); + await expect(launchButton).toHaveAttribute('data-state', 'warn'); + await expect(page.locator('#status-message')).toContainText(/limited/i); +}); + +test('ai generates fallback imagery and applies theme commands', async ({ page }) => { + await page.goto('/index.html'); + + await page.evaluate(() => { + window.__unityLandingTestHooks?.initialize(); + window.__unityLandingTestHooks?.markAllDependenciesReady(); + }); + + const launchButton = page.getByRole('button', { name: 'Talk to Unity' }); + await expect(launchButton).toBeEnabled(); + + await page.goto('/AI/index.html'); + + await page.waitForFunction(() => Boolean(window.__unityTestHooks?.isAppReady())); + + await page.unroute('https://text.pollinations.ai/openai'); + + const fallbackResponse = { + choices: [ + { + message: { + content: + '[command: theme_light]\nLet me paint a tranquil sunrise above misty mountains.' + } + } + ] + }; + + await page.route('https://text.pollinations.ai/openai', async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(fallbackResponse) + }); + }); + + const result = await page.evaluate(async () => { + const hooks = window.__unityTestHooks; + if (!hooks) { + throw new Error('Unity test hooks are not available'); + } + + const response = await hooks.sendUserInput( + 'Please paint a tranquil sunrise above misty mountains' + ); + + return { + response, + theme: document.body.dataset.theme, + heroState: document.getElementById('hero-stage')?.dataset.state ?? '', + heroUrl: hooks.getCurrentHeroImage(), + speakCalls: window.speechSynthesis.speakCalls.slice() + }; + }); + + expect(result.response?.commands).toContain('theme_light'); + expect(result.response?.imageUrl).toContain('image.pollinations.ai'); + expect(result.theme).toBe('light'); + expect(['loaded', 'error']).toContain(result.heroState); + if (result.heroState === 'loaded') { + expect(result.heroUrl).toContain('image.pollinations.ai'); + } + expect(result.speakCalls.some((entry) => /tranquil sunrise/i.test(entry))).toBe(true); +}); + +test('ai copies generated imagery when commanded by the assistant', async ({ page }) => { + await page.goto('/index.html'); + + await page.evaluate(() => { + window.__unityLandingTestHooks?.initialize(); + window.__unityLandingTestHooks?.markAllDependenciesReady(); + }); + + const launchButton = page.getByRole('button', { name: 'Talk to Unity' }); + await expect(launchButton).toBeEnabled(); + + await page.goto('/AI/index.html'); + + await page.waitForFunction(() => Boolean(window.__unityTestHooks?.isAppReady())); + + await page.unroute('https://text.pollinations.ai/openai'); + + const copyResponse = { + choices: [ + { + message: { + content: + '[command: copy_image]\nHere is your vibrant skyline.\nImage URL: https://image.pollinations.ai/prompt/copy-test.png' + } + } + ] + }; + + await page.route('https://text.pollinations.ai/openai', async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(copyResponse) + }); + }); + + const result = await page.evaluate(async () => { + const hooks = window.__unityTestHooks; + if (!hooks) { + throw new Error('Unity test hooks are not available'); + } + + const response = await hooks.sendUserInput( + 'Generate a futuristic skyline image for me.' + ); + + const history = hooks.getChatHistory(); + + return { + response, + clipboardWrites: window.__testClipboardWrites?.length ?? 0, + heroState: document.getElementById('hero-stage')?.dataset.state ?? '', + heroUrl: hooks.getCurrentHeroImage(), + lastMessage: history.at(-1) ?? null, + speakCalls: window.speechSynthesis.speakCalls.slice() + }; + }); + + expect(result.response?.commands).toContain('copy_image'); + expect(result.response?.imageUrl).toContain('image.pollinations.ai'); + expect(result.clipboardWrites).toBeGreaterThan(0); + expect(['loaded', 'error']).toContain(result.heroState); + if (result.heroState === 'loaded') { + expect(result.heroUrl).toContain('image.pollinations.ai'); + } + expect(result.lastMessage?.content ?? '').not.toMatch(/command/i); + expect(result.speakCalls.some((entry) => /image copied to clipboard/i.test(entry))).toBe(true); }); test('user can launch Talk to Unity and receive AI response with image and speech', async ({ page }) => { @@ -225,10 +414,7 @@ test('user can launch Talk to Unity and receive AI response with image and speec await expect(launchButton).toBeEnabled(); - await Promise.all([ - page.waitForNavigation({ url: /\/AI\/index\.html$/i }), - launchButton.click() - ]); + await page.goto('/AI/index.html'); await expect(page).toHaveURL(/\/AI\/index\.html$/i);