diff --git a/README.md b/README.md index 0ba22c5..adea2d0 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Nothing to compile if you want to use the extension - Follow the instructions to ##### Firefox * download or clone the repo * go to about:debugging#/runtime/this-firefox -* Click _"Load Temporary Add-on..."_ +* Click "Load Temporary Add-on..." * Browse to the `dist` folder with the extension and select the `manifest.json` file #### Usage without Extension @@ -42,3 +42,5 @@ This humble _hack-speriment_ would not be possible without the following project * [Virtual Browser Camera shaders](https://github.com/spite/virtual-webcam) by [spite](https://github.com/spite) * [Pose Animator](https://github.com/yemount/pose-animator) by [yemount](https://github.com/yemount) * [Meething](https://us.meething.space) by [Team Meething](https://github.com/meething/meething/graphs/contributors) + +![image](https://user-images.githubusercontent.com/1423657/82818656-561dbe80-9e9f-11ea-90a1-5436fdcb84e5.png) diff --git a/js/filter-stream.js b/js/filter-stream.js index 1636b7d..5cc8b1e 100644 --- a/js/filter-stream.js +++ b/js/filter-stream.js @@ -7,58 +7,65 @@ class FilterStream { this.stream = stream; const video = document.createElement("video"); const canvas = document.createElement("canvas"); - //const svg = document.querySelector(".illustration-canvas"); this.canvas = canvas; - //this.svg = svg; - - video.srcObject = stream; - video.autoplay = true; - this.video = video; - //this.ctx = this.svg.getContext("2d"); - this.outputStream = this.canvas.captureStream(); - this.canvasActiveLayer = null; - this.poseEmitter = new PoseEmitter(this.video, this.video.videoWidth, this.video.videoHeight, false) + this.addedCanvas = false; - video.addEventListener("playing", () => { + this.svgCanvas; + + video.addEventListener("playing", async () => { + console.log('video is playing',this.video.videoWidth, this.video.videoHeight); // Use a 2D Canvas. this.canvas.width = this.video.videoWidth; this.canvas.height = this.video.videoHeight; + if (!this.poseEmitter) this.poseEmitter = new PoseEmitter(this.video, this.video.videoWidth, this.video.videoHeight) this.update(); }); + + video.srcObject = stream; + video.autoplay = true; + this.video = video; + + this.ctx = this.canvas.getContext("2d"); + this.outputStream = this.canvas.captureStream(); + } - update() { + async update() { + if(!this.poseEmitter) return; // Use a 2D Canvas - // this.ctx.filter = 'invert(1)'; - // this.canvas.width = this.video.videoWidth; - // this.canvas.height = this.video.videoHeight; - // this.svg.width = this.video.videoWidth; - // this.svg.height = this.video.videoHeight; - - // this.ctx.drawImage(this.video, 0, 0); - // this.ctx.fillStyle = "#ff00ff"; - // this.ctx.textBaseline = "top"; - // this.ctx.fillText("Virtual", 10, 10); - this.drawOnCanvas(); - + //if (this.svgCanvas instanceof HTMLCanvasElement) { + if (this.svgCanvas) { + this.canvas.width = this.video.videoWidth; + this.canvas.height = this.video.videoHeight; + this.ctx.drawImage(this.svgCanvas, 0, 0, this.video.videoHeight, this.video.videoWidth); + //this.ctx.fillStyle = "#ffff00"; + //this.ctx.textBaseline = "top"; + //this.ctx.fillText("Virtual Camera", 10, 10); + } else { + //this.canvas.width = this.video.videoWidth; + //this.canvas.height = this.video.videoHeight; + //this.ctx.drawImage(this.video, 0, 0); + //this.ctx.fillStyle = "#ff00ff"; + //this.ctx.textBaseline = "top"; + //this.ctx.fillText("Loading...", 10, 10); + } + await this.drawOnCanvas(); requestAnimationFrame(() => this.update()); } - + + async drawOnCanvas() { - let svgCanvas = await this.poseEmitter.sampleAndDetect(); - if(svgCanvas instanceof HTMLCanvasElement){ + this.svgCanvas = await this.poseEmitter.sampleAndDetect(); + if(this.svgCanvas instanceof HTMLCanvasElement){ + this.svgCanvas.width = this.video.videoWidth; + this.svgCanvas.height = this.video.videoHeight; if(!this.addedCanvas){ - document.body.appendChild(svgCanvas); + this.outputStream = this.svgCanvas.captureStream(); this.addedCanvas = true; } - // console.log("SVG invisible canvas - ", svgCanvas); - // let svgImage = svgCanvas.getContext("2d").createImageData(svgCanvas.width, svgCanvas.height); - // this.ctx.drawImage(svgImage, 0, 0); - // TODO: REPLACE INPUT WITH DRIVER VIDEO AND OUTPUT CANVAS WITH SVG CANVAS } } - } export { FilterStream }; diff --git a/js/main.js b/js/main.js index bf0a526..bc52359 100644 --- a/js/main.js +++ b/js/main.js @@ -1,21 +1,14 @@ import { monkeyPatchMediaDevices } from "./media-devices.js"; + monkeyPatchMediaDevices(); -//import PoseDetector from "./poseDetection.js"; async function init() { const res = await navigator.mediaDevices.enumerateDevices(); console.log(res); const stream = await navigator.mediaDevices.getUserMedia({ - video: { deviceId: "virtual", width: 320, height: 240 }, + video: { deviceId: "virtual" }, audio: false }); - // const video = document.createElement("video"); - // video.setAttribute("id", "local"); - // video.setAttribute("class", "video-local"); - // video.srcObject = stream; - // video.autoplay = true; - // document.body.append(video); - //new PoseDetector(); } init(); diff --git a/js/media-devices.js b/js/media-devices.js index f2be496..ad45683 100644 --- a/js/media-devices.js +++ b/js/media-devices.js @@ -3,6 +3,7 @@ import { FilterStream } from "./filter-stream.js"; function monkeyPatchMediaDevices() { const enumerateDevicesFn = MediaDevices.prototype.enumerateDevices; const getUserMediaFn = MediaDevices.prototype.getUserMedia; + var filter; MediaDevices.prototype.enumerateDevices = async function() { const res = await enumerateDevicesFn.call(navigator.mediaDevices); @@ -24,15 +25,14 @@ function monkeyPatchMediaDevices() { args[0].video.deviceId === "virtual" || args[0].video.deviceId.exact === "virtual" ) { + if(filter && filter.outputStream) return filter.outputStream; // This constraints could mimick closely the request. // Also, there could be a preferred webcam on the options. // Right now it defaults to the predefined input. const constraints = { video: { - facingMode: args[0].facingMode, - advanced: args[0].video.advanced, - width: args[0].video.width, - height: args[0].video.height + width: args[0].video.width || 640, + height: args[0].video.height || 480 }, audio: false }; @@ -42,7 +42,7 @@ function monkeyPatchMediaDevices() { ); if (res) { var shader = false; - const filter = new FilterStream(res, shader); + filter = new FilterStream(res, shader); return filter.outputStream; } } diff --git a/js/poseDetection.js b/js/poseDetection.js index c3d4a2f..8150414 100644 --- a/js/poseDetection.js +++ b/js/poseDetection.js @@ -14,7 +14,7 @@ import * as paper from "paper"; import "babel-polyfill"; import * as boySVG from "../svg/boy.svg"; -import * as girlSVG from "../svg/girl.svg"; +//import * as girlSVG from "../svg/girl.svg"; export default class PoseDetector { constructor(video,videoWidth,videoHeight) { diff --git a/js/poseEmitter.js b/js/poseEmitter.js index 154645a..53c0adf 100644 --- a/js/poseEmitter.js +++ b/js/poseEmitter.js @@ -1,8 +1,8 @@ +import "babel-polyfill"; import * as posenet from '@tensorflow-models/posenet'; import * as facemesh from '@tensorflow-models/facemesh'; import * as tf from '@tensorflow/tfjs'; import * as paper from "paper"; -import "babel-polyfill"; import { drawKeypoints, @@ -19,15 +19,14 @@ import { FileUtils } from "./utils/fileUtils.js"; import * as boySVG from "../svg/boy.svg"; -import * as girlSVG from "../svg/girl.svg"; export default class PoseEmitter { - constructor(video, videoWidth, videoHeight, callback) { + constructor(video, videoWidth, videoHeight) { this.video = video ? video : document.getElementById("local"); // Camera stream video element - this.videoWidth = videoWidth ? videoWidth : 320; - this.videoHeight = videoHeight ? videoHeight : 240; + this.videoWidth = videoWidth ? videoWidth : 640; + this.videoHeight = videoHeight ? videoHeight : 480; // Canvas this.faceDetection = null; @@ -36,13 +35,13 @@ export default class PoseEmitter { // let canvas = document.querySelector(".illustration-canvas"); // TODO: use an invisible canvas we return at the end, do not render it this.canvas = document.createElement('canvas'); - this.canvas.width = videoWidth ? videoWidth : 320; - this.canvas.height = videoHeight ? videoHeight : 240; + this.canvas.width = videoWidth ? videoWidth : 640; + this.canvas.height = videoHeight ? videoHeight : 480; this.canvasScope.setup(this.canvas); this.canvasWidth = this.canvas.width; this.canvasHeight = this.canvas.height; - console.log("Canvas scope = ", this.canvasScope); + // console.log("Canvas scope = ", this.canvasScope); // ML models @@ -122,16 +121,7 @@ export default class PoseEmitter { scoreThreshold: self.minPartConfidence, nmsRadius: self.nmsRadius }); - console.log("pose detected : ", all_poses); - - //Dispatch event - // var event = new CustomEvent("poseDetected", { - // detail: { - // faceMesh: self.faceDetection, - // pose: all_poses - // } - // }); - // self.video.dispatchEvent(event); + // console.log("pose detected : ", all_poses); poses = poses.concat(all_poses); input.dispose(); @@ -139,7 +129,7 @@ export default class PoseEmitter { self.canvasScope.project.clear(); if (poses.length >= 1 && self.illustration) { - //Skeleton.flipPose(poses[0]); + Skeleton.flipPose(poses[0]); if (self.faceDetection && self.faceDetection.length > 0) { let face = Skeleton.toFaceFrame(self.faceDetection[0]); @@ -165,11 +155,6 @@ export default class PoseEmitter { console.log("ERROR! Paper project undefined", self.canvasScope); } - // Send the activelayer to the callback function - // if(callback){ - // callback(self.canvasScope.project.activeLayer); - // } - } catch (err) { // input.dispose(); console.log(err); diff --git a/package-lock.json b/package-lock.json index ff61bb6..6a0af2a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "meething-ml-camera", - "version": "1.0.0", + "version": "1.0.1", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -2318,6 +2318,25 @@ "object-assign": "^4.0.1" } }, + "file-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/file-match/-/file-match-1.0.2.tgz", + "integrity": "sha1-ycrSZdLIrfOoFHWw30dYWQafrvc=", + "dev": true, + "requires": { + "utils-extend": "^1.0.6" + } + }, + "file-system": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/file-system/-/file-system-2.2.2.tgz", + "integrity": "sha1-fWWDPjojR9zZVqgTxncVPtPt2Yc=", + "dev": true, + "requires": { + "file-match": "^1.0.1", + "utils-extend": "^1.0.4" + } + }, "find-up": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", @@ -10148,6 +10167,17 @@ } } }, + "parcel-plugin-static-files-copy": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/parcel-plugin-static-files-copy/-/parcel-plugin-static-files-copy-2.3.1.tgz", + "integrity": "sha512-yqB1bhSK+hbfxSjc1y/gBc+Fm6bedNrofx75wgnI0sP+6oEBqjyN51tlJVLu6pZhBLi11ZFwAM2XubCUh2G0+A==", + "dev": true, + "requires": { + "file-system": "2.2.2", + "minimatch": "3.0.4", + "path": "0.12.7" + } + }, "parse-json": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", @@ -10157,6 +10187,16 @@ "error-ex": "^1.2.0" } }, + "path": { + "version": "0.12.7", + "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", + "integrity": "sha1-1NwqUGxM4hl+tIHr/NWzbAFAsQ8=", + "dev": true, + "requires": { + "process": "^0.11.1", + "util": "^0.10.3" + } + }, "path-exists": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", @@ -10861,6 +10901,12 @@ "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==", "dev": true }, + "process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=", + "dev": true + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -11567,6 +11613,23 @@ "os-homedir": "^1.0.0" } }, + "util": { + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz", + "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==", + "dev": true, + "requires": { + "inherits": "2.0.3" + }, + "dependencies": { + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", + "dev": true + } + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -11585,6 +11648,12 @@ "object.getownpropertydescriptors": "^2.1.0" } }, + "utils-extend": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/utils-extend/-/utils-extend-1.0.8.tgz", + "integrity": "sha1-zP17ZFQPjpDuIe7Fd2nQZRyril8=", + "dev": true + }, "validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", diff --git a/package.json b/package.json index 27cf0f8..cdb8da0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "meething-ml-camera", - "version": "1.0.0", + "version": "1.0.1", "description": "", "author": "", "license": "(MIT OR Apache-2.0)", @@ -17,7 +17,8 @@ }, "scripts": { "start": "parcel cam.html --no-source-maps", - "build": "NODE_ENV=production parcel build cam.html --no-source-maps -o index.html", + "build": "rm -rf ./dist && NODE_ENV=production parcel build js/main.js --no-source-maps -o main.js && npm run patch", + "patch": "sed -i 's|/boy..*.svg|//cdn.jsdelivr.net/gh/meething/meething-ml-camera@ext/svg/boy.svg|g' ./dist/main.js", "lint": "eslint .", "link-local": "yalc link" }, @@ -39,6 +40,7 @@ "eslint": "^4.19.1", "eslint-config-google": "^0.9.1", "parcel-bundler": "~1.12.4", + "parcel-plugin-static-files-copy": "^2.3.1", "yalc": "~1.0.0-pre.27" }, "eslintConfig": { diff --git a/static/boy.svg b/static/boy.svg new file mode 100644 index 0000000..2a32a1c --- /dev/null +++ b/static/boy.svg @@ -0,0 +1,221 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/static/girl.svg b/static/girl.svg new file mode 100644 index 0000000..e6356ad --- /dev/null +++ b/static/girl.svg @@ -0,0 +1,263 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/icon-128.png b/static/icon-128.png similarity index 100% rename from icon-128.png rename to static/icon-128.png diff --git a/icon-16.png b/static/icon-16.png similarity index 100% rename from icon-16.png rename to static/icon-16.png diff --git a/icon-48.png b/static/icon-48.png similarity index 100% rename from icon-48.png rename to static/icon-48.png diff --git a/icon-64.png b/static/icon-64.png similarity index 100% rename from icon-64.png rename to static/icon-64.png diff --git a/icon.png b/static/icon.png similarity index 100% rename from icon.png rename to static/icon.png diff --git a/js/inject.js b/static/inject.js similarity index 66% rename from js/inject.js rename to static/inject.js index c713ebe..50ef014 100644 --- a/js/inject.js +++ b/static/inject.js @@ -1,8 +1,7 @@ "use strict"; const script = document.createElement("script"); -script.setAttribute("type", "module"); -script.setAttribute("src", chrome.extension.getURL("js/main.js")); +script.setAttribute("src", chrome.extension.getURL("main.js")); const head = document.head || document.getElementsByTagName("head")[0] || diff --git a/manifest.json b/static/manifest.json similarity index 89% rename from manifest.json rename to static/manifest.json index e0895e3..646a8d2 100644 --- a/manifest.json +++ b/static/manifest.json @@ -1,6 +1,6 @@ { "name": "Meething Virtual Webcam", - "version": "1.0.0", + "version": "1.0.1", "minimum_chrome_version": "10.0", "icons": { "16": "icon-16.png", @@ -14,7 +14,7 @@ "" ], "js": [ - "dist/inject.js" + "inject.js" ], "run_at": "document_start", "all_frames": true @@ -26,6 +26,6 @@ ], "manifest_version": 2, "web_accessible_resources": [ - "dist/*" + "*" ] }