From cb1ea2cd668dc21c029cd495144c0ed51a153bd9 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Tue, 22 Aug 2023 07:42:16 -0500 Subject: [PATCH 01/79] Remote Display proof-of-concept --- Cargo.lock | 156 ++++++++++- contrib/remote-display-server.ts | 131 +++++++++ contrib/remote-display-webext/background.js | 280 ++++++++++++++++++++ contrib/remote-display-webext/manifest.json | 14 + crates/core/Cargo.toml | 5 + crates/core/src/geom.rs | 9 +- crates/core/src/gesture.rs | 4 +- crates/core/src/input.rs | 4 +- crates/core/src/settings/mod.rs | 16 ++ crates/core/src/view/common.rs | 2 + crates/core/src/view/mod.rs | 3 + crates/core/src/view/remote_display/mod.rs | 258 ++++++++++++++++++ crates/emulator/src/main.rs | 4 + crates/plato/src/app.rs | 4 + 14 files changed, 880 insertions(+), 10 deletions(-) create mode 100644 contrib/remote-display-server.ts create mode 100644 contrib/remote-display-webext/background.js create mode 100644 contrib/remote-display-webext/manifest.json create mode 100644 crates/core/src/view/remote_display/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 59624af4..10bd493a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,6 +148,12 @@ version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +[[package]] +name = "bytemuck" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" + [[package]] name = "byteorder" version = "1.4.3" @@ -220,6 +226,12 @@ dependencies = [ "inout", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -274,6 +286,10 @@ name = "deranged" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +name = "data-encoding" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "digest" @@ -429,6 +445,7 @@ dependencies = [ "futures-core", "futures-io", "futures-macro", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -464,6 +481,17 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.28.0" @@ -634,6 +662,19 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.24.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f3dfdbdd72063086ff443e297b61695500514b1e41095b6fb9a5ab48a70a711" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "num-rational", + "num-traits", +] + [[package]] name = "importer" version = "0.9.39" @@ -798,6 +839,27 @@ dependencies = [ "libc", ] +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.16" @@ -869,9 +931,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" [[package]] name = "pin-utils" @@ -904,9 +966,11 @@ dependencies = [ "entities", "filetime", "flate2", + "futures-util", "fxhash", "globset", - "indexmap 2.0.0", + "image", + "indexmap 1.9.3", "kl-hyphenate", "lazy_static", "levenshtein", @@ -923,8 +987,11 @@ dependencies = [ "serde_json", "thiserror", "titlecase", + "tokio", + "tokio-tungstenite", "toml", "unicode-normalization", + "url", "walkdir", "xi-unicode", "zip", @@ -943,6 +1010,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "proc-macro2" version = "1.0.67" @@ -961,11 +1034,35 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] [[package]] name = "rand_xoshiro" @@ -1279,9 +1376,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" dependencies = [ "libc", "windows-sys", @@ -1386,9 +1483,21 @@ dependencies = [ "num_cpus", "pin-project-lite", "socket2 0.5.4", + "tokio-macros", "windows-sys", ] +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -1399,6 +1508,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2dbec703c26b00d74844519606ef15d09a7d6857860f84ad223dec002ddea2" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.8" @@ -1479,6 +1600,25 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "tungstenite" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e862a1c4128df0112ab625f55cd5c934bcb4312ba80b39ae4b4835a3fd58e649" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "sha1", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "typenum" version = "1.16.0" @@ -1529,6 +1669,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "version-compare" version = "0.1.1" diff --git a/contrib/remote-display-server.ts b/contrib/remote-display-server.ts new file mode 100644 index 00000000..75f47f8a --- /dev/null +++ b/contrib/remote-display-server.ts @@ -0,0 +1,131 @@ +import { Application, Router } from "https://deno.land/x/oak@v12.6.0/mod.ts"; +import { Foras, zlib } from "https://deno.land/x/foras@2.0.8/src/deno/mod.ts"; +import $ from "https://deno.land/x/dax@0.34.0/mod.ts"; + +await Foras.initBundledOnce(); + +// #region support functions +const bytes = Intl.NumberFormat("en", { + notation: "compact", + style: "unit", + unit: "byte", + unitDisplay: "narrow", +}); + +/** Use ImageMagick `convert` to create a deflated PBM. */ +async function convertToBlob( + data: Uint8Array, + width: number, + height: number, +): Promise { + const start = performance.now(); + const image = + await $`convert - -resize ${width}x${height}! -dither FloydSteinberg -remap pattern:gray50 pnm:-` + .stdin(data) + .bytes(); + const compressed = zlib(image); + console.log( + `converted ${bytes.format(data.byteLength)} frame to ${ + bytes.format(compressed.byteLength) + } (${bytes.format(image.byteLength)} inflated) in ${ + Math.round(performance.now() - start) + }ms`, + ); + return new Blob([compressed], { type: "application/x-deflate" }); +} +// #endregion + +const app = new Application(); +const port = parseInt(Deno.env.get("PORT") || "8222"); +const router = new Router(); + +let deviceSocket: WebSocket | undefined; +let browserSocket: WebSocket | undefined; +let deviceWidth = 0; +let deviceHeight = 0; + +// #region browser socket +router.get("/browser", (ctx) => { + if (browserSocket) { + ctx.response.status = 400; + ctx.response.body = "Only one browser connection allowed"; + return; + } + browserSocket = ctx.upgrade(); + console.log("Browser starting connection"); + + browserSocket.onopen = () => { + console.log("Browser connected"); + if (!deviceSocket || !deviceWidth || !deviceHeight) return; + browserSocket?.send( + JSON.stringify({ + type: "size", + value: { width: deviceWidth, height: deviceHeight }, + }), + ); + }; + + browserSocket.onclose = () => { + browserSocket = undefined; + console.log("Browser disconnected"); + }; + + browserSocket.onmessage = (m) => { + if (m.data instanceof ArrayBuffer) { + console.log("Browser sends image"); + if (!deviceSocket) return; + convertToBlob(new Uint8Array(m.data), deviceWidth, deviceHeight) + .then((blob) => deviceSocket?.send(blob)); + return; + } + console.log("Browser sends: ", m.data); + deviceSocket?.send(m.data); + }; +}); +// #endregion + +// #region device socket +router.get("/device", (ctx) => { + if (deviceSocket) { + ctx.response.status = 400; + ctx.response.body = "Only one device connection allowed"; + return; + } + deviceSocket = ctx.upgrade(); + console.log("Device starting connection"); + + deviceSocket.onopen = () => { + console.log("Device connected"); + }; + + deviceSocket.onclose = () => { + deviceSocket = undefined; + console.log("Device disconnected"); + }; + + deviceSocket.onmessage = (m) => { + if (m.data instanceof ArrayBuffer) return; + console.log("Device sends: ", m.data); + const msg = JSON.parse(m.data); + switch (msg.type) { + case "size": + deviceWidth = msg.value.width; + deviceHeight = msg.value.height; + $`convert -size ${deviceWidth}x${deviceHeight} xc:white -gravity center -font DejaVu-Sans-Mono-Book -pointsize 32 -annotate +0+0 "Welcome" pnm:-` + .bytes() + .then((data) => convertToBlob(data, deviceWidth, deviceHeight)) + .then((blob) => deviceSocket?.send(blob)); + browserSocket?.send(m.data); + break; + default: + browserSocket?.send(m.data); + } + }; +}); +// #endregion + +app.use(router.routes()); +app.use(router.allowedMethods()); + +console.log("Listening at port " + port); +await app.listen({ port }); diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js new file mode 100644 index 00000000..af9bff0a --- /dev/null +++ b/contrib/remote-display-webext/background.js @@ -0,0 +1,280 @@ +// #region browser interactivity +async function tabOffset(offset) { + const tabs = await browser.tabs.query({ currentWindow: true }); + const currentTab = tabs.find((tab) => tab.active); + const currentIndex = tabs.indexOf(currentTab); + const newIndex = (currentIndex + offset + tabs.length) % tabs.length; + await browser.tabs.update(tabs[newIndex].id, { active: true }); +} + +async function windowOffset(offset) { + const windows = await browser.windows.getAll({ populate: true }); + const currentWindow = windows.find((window) => window.focused); + const currentIndex = windows.indexOf(currentWindow); + const newIndex = (currentIndex + offset + windows.length) % windows.length; + await browser.windows.update(windows[newIndex].id, { focused: true }); + await new Promise((resolve) => setTimeout(resolve, 100)); +} + +async function currentTab() { + const tabs = await browser.tabs.query({ currentWindow: true }); + return tabs.find((tab) => tab.active); +} + +async function currentTabInfo() { + const tabs = await browser.tabs.query({ currentWindow: true }); + const currentTab = tabs.find((tab) => tab.active); + const currentTabIndex = tabs.indexOf(currentTab); + const windows = await browser.windows.getAll({ populate: true }); + const currentWindow = windows.find((window) => window.focused); + const currentWindowIndex = windows.indexOf(currentWindow); + const url = new URL(currentTab.url); + return `W${currentWindowIndex + 1} T${ + currentTabIndex + 1 + }/${tabs.length} ${url.host}`; +} + +async function scroll(pctX, pctY, pct) { + await browser.tabs.executeScript({ + code: `(() => { + const el = [...document.elementsFromPoint( + window.innerWidth * ${pctX}, window.innerHeight * ${pctY} + )].find((e) => Math.abs(e.scrollHeight - e.clientHeight) > 10); + const prevTop = el?.scrollTop; + el?.scrollBy(0, window.innerHeight * ${pct}); + if (!el || el?.scrollTop === prevTop) { + window.scrollBy(0, window.innerHeight * ${pct}); + } + })()`, + }); +} + +async function zoomPage(addFactor) { + const factor = await browser.tabs.getZoom(); + let newFactor = factor + addFactor; + if (newFactor < 0.3) newFactor = 0.3; + if (newFactor > 5) newFactor = 5; + await browser.tabs.setZoom(undefined, newFactor); +} + +async function goForward() { + await browser.tabs.goForward(); +} + +async function goBack() { + await browser.tabs.goBack(); +} + +async function closeCurrentTab() { + await browser.tabs.remove((await currentTab()).id); +} + +async function reloadCurrentTab() { + await browser.tabs.reload(); +} + +async function resizeViewport(width, height) { + const window = await browser.windows.getCurrent(); + const tab = await currentTab(); + if (tab.width === width && tab.height === height) return; + if (tab.width > width || tab.height > height) { + await browser.windows.update(window.id, { + width: width, + height: height, + }); + await resizeViewport(width, height); + } + const offsetWidth = window.width - tab.width; + const offsetHeight = window.height - tab.height; + await browser.windows.update(window.id, { + width: width + offsetWidth, + height: height + offsetHeight, + }); + await new Promise((resolve) => setTimeout(resolve, 100)); +} + +async function openLinkUnderTap(pctX, pctY) { + const tab = await currentTab(); + const [url] = await browser.tabs.executeScript({ + code: + `[...document.elementsFromPoint(window.innerWidth * ${pctX}, window.innerHeight * ${pctY})] + .find((e) => !!e.href) + ?.href`, + }); + if (!url) return; + await browser.tabs.create({ url }); + await browser.tabs.update(tab.id, { active: true }); + return new URL(url).host; +} + +async function clickUnderTap(pctX, pctY) { + await browser.tabs.executeScript({ + code: + `document.elementFromPoint(window.innerWidth * ${pctX}, window.innerHeight * ${pctY})?.click()`, + }); +} +// #endregion + +// #region main loop +async function getWsUrl() { + let wsUrl = "ws://localhost:8222/browser"; + const result = await browser.storage.local.get("wsUrl"); + if (result.wsUrl) { + wsUrl = result.wsUrl; + } + return wsUrl; +} + +let deviceWidth = 0; +let deviceHeight = 0; +let ws; + +async function sendImage() { + if (!ws || ws.readyState !== WebSocket.OPEN) return; + console.log("capturing"); + const dataUrl = await browser.tabs.captureVisibleTab(); + const buf = await (await fetch(dataUrl)).arrayBuffer(); + ws.send(buf); + await new Promise((resolve) => { + ws.addEventListener("message", (e) => { + const msg = JSON.parse(e.data); + if (msg.type === "displayUpdated") { + resolve(); + } + }, { once: true }); + }); +} + +function sendNotice(notice) { + if (!ws || ws.readyState !== WebSocket.OPEN) return; + ws.send(JSON.stringify({ type: "notify", value: notice })); +} + +let timeout; +browser.tabs.onUpdated.addListener(async (_id, changeInfo, tab) => { + if (!tab.active) return; + if (changeInfo.status !== "complete") return; + const myTab = await currentTab(); + if (tab.id !== myTab.id) return; + if (timeout) { + clearTimeout(timeout); + } + timeout = setTimeout(async () => { + console.log("updating from tab load", changeInfo, tab); + await sendImage(); + const info = await currentTabInfo(); + sendNotice(info); + }, 1000); +}); + +async function connect() { + console.log("attempting connection"); + if (ws) ws.close(); + ws = new WebSocket(await getWsUrl()); + ws.onmessage = async (e) => { + console.log("message", e); + const msg = JSON.parse(e.data); + switch (msg.type) { + case "size": { + const { width, height } = msg.value; + deviceWidth = width; + deviceHeight = height; + await sendImage(); + break; + } + case "swipe": { + const { dir, start, end } = msg.value; + switch (dir) { + case "north": + case "south": { + const dy = end.y - start.y; + await scroll( + start.x / deviceWidth, + start.y / deviceHeight, + -(dy / deviceHeight), + ); + await sendImage(); + break; + } + case "east": + case "west": { + const offset = dir === "east" ? -1 : 1; + const dx = end.x - start.x; + // change windows if swipe covers more than half the screen + await (Math.abs(dx) > deviceHeight / 2 + ? windowOffset(offset) + : tabOffset(offset)); + await sendImage(); + const info = await currentTabInfo(); + sendNotice(info); + break; + } + } + break; + } + case "arrow": { + const { dir } = msg.value; + switch (dir) { + case "east": + await goForward(); + break; + case "west": + await goBack(); + break; + case "north": { + const info = await currentTabInfo(); + sendNotice(`closing ${info}`); + await closeCurrentTab(); + await sendImage(); + const newInfo = await currentTabInfo(); + sendNotice(newInfo); + break; + } + } + break; + } + case "pinch": + await zoomPage(-0.1); + await sendImage(); + break; + case "spread": + await zoomPage(0.1); + await sendImage(); + break; + case "rotate": + await reloadCurrentTab(); + break; + case "holdFingerShort": + await resizeViewport(deviceWidth, deviceHeight); + await sendImage(); + await ws.send(JSON.stringify({ type: "refreshDisplay" })); + break; + case "holdFingerLong": { + const [{ x, y }] = msg.value; + const tab = await openLinkUnderTap(x / deviceWidth, y / deviceHeight); + if (tab) sendNotice(`${tab} opened`); + else sendNotice("no link under finger"); + break; + } + case "tap": { + const { x, y } = msg.value; + await clickUnderTap(x / deviceWidth, y / deviceHeight); + await sendImage(); + break; + } + } + }; +} + +async function mainLoop() { + for (;;) { + if (!ws || ws.readyState === WebSocket.CLOSED) { + console.log("connection closed, restarting"); + await connect(); + } + await new Promise((resolve) => setTimeout(resolve, 5000)); + } +} + +mainLoop().catch((e) => console.error(e)); +// #endregion diff --git a/contrib/remote-display-webext/manifest.json b/contrib/remote-display-webext/manifest.json new file mode 100644 index 00000000..4f2f6918 --- /dev/null +++ b/contrib/remote-display-webext/manifest.json @@ -0,0 +1,14 @@ +{ + "manifest_version": 2, + "name": "Plato Remote Display", + "version": "0.1", + "description": "Plato Remote Display", + "background": { + "scripts": ["background.js"] + }, + "permissions": [ + "tabs", + "", + "storage" + ] +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index d7bd07e0..819e7d31 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -40,3 +40,8 @@ rand_core = "0.6.4" rand_xoshiro = "0.6.0" percent-encoding = "2.3.0" chrono = { version = "0.4.30", features = ["serde", "clock"], default-features = false } +image = { version = "0.24.7", features = ["pnm"], default-features = false } +url = { version = "2.1.0" } +tokio-tungstenite = "0.20.0" +tokio = { version = "1.32.0", features = ["macros", "rt", "net", "sync"] } +futures-util = { version = "0.3.28", default-features = false, features = ["sink", "std"] } diff --git a/crates/core/src/geom.rs b/crates/core/src/geom.rs index 7a1c618e..0ce1c794 100644 --- a/crates/core/src/geom.rs +++ b/crates/core/src/geom.rs @@ -4,7 +4,8 @@ use std::cmp::Ordering; use std::f32::consts; use std::ops::{Add, AddAssign, Sub, SubAssign, Mul, MulAssign, Div, DivAssign}; -#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] pub enum Dir { North, East, @@ -41,7 +42,8 @@ impl fmt::Display for Dir { } } -#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] pub enum DiagDir { NorthWest, NorthEast, @@ -71,7 +73,8 @@ impl fmt::Display for DiagDir { } } -#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] pub enum Axis { Horizontal, Vertical, diff --git a/crates/core/src/gesture.rs b/crates/core/src/gesture.rs index 07c6d1a9..39d45970 100644 --- a/crates/core/src/gesture.rs +++ b/crates/core/src/gesture.rs @@ -5,6 +5,7 @@ use fxhash::FxHashMap; use std::f64; use std::time::Duration; use std::thread; +use serde::Serialize; use crate::unit::mm_to_px; use crate::input::{DeviceEvent, FingerStatus, ButtonCode, ButtonStatus}; use crate::view::Event; @@ -16,7 +17,8 @@ pub const HOLD_JITTER_MM: f32 = 1.5; pub const HOLD_DELAY_SHORT: Duration = Duration::from_millis(666); pub const HOLD_DELAY_LONG: Duration = Duration::from_millis(1333); -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, Serialize)] +#[serde(rename_all = "camelCase", tag = "type", content = "value")] pub enum GestureEvent { Tap(Point), MultiTap([Point; 2]), diff --git a/crates/core/src/input.rs b/crates/core/src/input.rs index 34bc3437..aa1fd904 100644 --- a/crates/core/src/input.rs +++ b/crates/core/src/input.rs @@ -8,6 +8,7 @@ use std::sync::mpsc::{self, Sender, Receiver}; use std::os::unix::io::AsRawFd; use std::ffi::CString; use fxhash::FxHashMap; +use serde::Serialize; use crate::framebuffer::Display; use crate::settings::ButtonScheme; use crate::device::CURRENT_DEVICE; @@ -130,7 +131,8 @@ impl ButtonStatus { } } -#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash, Serialize)] +#[serde(rename_all = "camelCase")] pub enum ButtonCode { Power, Home, diff --git a/crates/core/src/settings/mod.rs b/crates/core/src/settings/mod.rs index 5f16f854..5d4fb411 100644 --- a/crates/core/src/settings/mod.rs +++ b/crates/core/src/settings/mod.rs @@ -121,6 +121,7 @@ pub struct Settings { pub reader: ReaderSettings, pub import: ImportSettings, pub dictionary: DictionarySettings, + pub remote_display: RemoteDisplaySettings, pub sketch: SketchSettings, pub calculator: CalculatorSettings, pub battery: BatterySettings, @@ -193,6 +194,20 @@ impl Default for DictionarySettings { } } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, rename_all = "kebab-case")] +pub struct RemoteDisplaySettings { + pub address: String +} + +impl Default for RemoteDisplaySettings { + fn default() -> Self { + RemoteDisplaySettings { + address: "ws://localhost:8222/device".to_string() + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, rename_all = "kebab-case")] pub struct SketchSettings { @@ -546,6 +561,7 @@ impl Default for Settings { reader: ReaderSettings::default(), import: ImportSettings::default(), dictionary: DictionarySettings::default(), + remote_display: RemoteDisplaySettings::default(), sketch: SketchSettings::default(), calculator: CalculatorSettings::default(), battery: BatterySettings::default(), diff --git a/crates/core/src/view/common.rs b/crates/core/src/view/common.rs index e4a5a696..94480fee 100644 --- a/crates/core/src/view/common.rs +++ b/crates/core/src/view/common.rs @@ -85,6 +85,8 @@ pub fn toggle_main_menu(view: &mut dyn View, rect: Rectangle, enable: Option>) } #[derive(Debug, Clone, Eq, PartialEq)] @@ -375,6 +377,7 @@ pub enum AppCmd { }, TouchEvents, RotationValues, + RemoteDisplay } #[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)] diff --git a/crates/core/src/view/remote_display/mod.rs b/crates/core/src/view/remote_display/mod.rs new file mode 100644 index 00000000..25fc9699 --- /dev/null +++ b/crates/core/src/view/remote_display/mod.rs @@ -0,0 +1,258 @@ +use crate::context::Context; +use crate::font::Fonts; +use crate::framebuffer::{Framebuffer, Pixmap, UpdateMode}; +use crate::geom::{Dir, Rectangle}; +use crate::gesture::GestureEvent; +use crate::view::{Bus, Event, Hub, RenderData, RenderQueue, View}; +use crate::view::{Id, ID_FEEDER}; +use anyhow::Error; +use flate2::bufread::ZlibDecoder; +use futures_util::{SinkExt, StreamExt}; +use image::codecs::pnm::PnmDecoder; +use image::ImageDecoder; +use serde::Deserialize; +use serde_json::{json, Value}; +use std::io::Read; +use std::sync::mpsc as std_mpsc; +use std::thread::spawn; +use tokio::sync::mpsc as tokio_mpsc; +use tokio_tungstenite::{connect_async, tungstenite::protocol::Message}; +use url::Url; + +#[derive(Debug, Clone)] +enum SocketEvent { + Finished, + SendJSON(Value), +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "type", content = "value", rename_all = "camelCase")] +enum ServerMessage { + Notify(String), + RefreshDisplay, +} + +#[tokio::main(flavor = "current_thread")] +async fn display_connection( + event_tx: std_mpsc::Sender, + mut socket_rx: tokio_mpsc::Receiver, + url: Url, +) -> Result<(), Box> { + let (mut socket, _) = connect_async(url).await?; + event_tx.send(Event::Notify("Connected".to_string()))?; + loop { + tokio::select! { + Some(event) = socket_rx.recv() => { + match event { + SocketEvent::Finished => { break; } + SocketEvent::SendJSON(val) => { + socket.send(Message::Text(val.to_string())).await?; + } + } + } + Some(msg) = socket.next() => { + let text = match msg { + Ok(Message::Text(text)) => text, + Ok(Message::Binary(bin)) => { + event_tx.send(Event::UpdateRemoteView(Box::new(bin)))?; + continue; + }, + Ok(Message::Close(_)) => { break; } + Ok(_) => { continue; } + Err(e) => { + event_tx.send(Event::Notify(e.to_string()))?; + break; + } + }; + + let server_msg = match serde_json::from_str::(&text) { + Ok(sm) => sm, + Err(e) => { + println!("{}", e); + println!("contents: {}", text); + event_tx.send(Event::Notify("Invalid message from server".to_string()))?; + continue; + } + }; + + match server_msg { + ServerMessage::Notify(msg) => { + event_tx.send(Event::Notify(msg))?; + } + ServerMessage::RefreshDisplay => { + event_tx.send(Event::Update(UpdateMode::Full))?; + } + } + } + } + } + event_tx.send(Event::Notify("Disconnected".to_string()))?; + socket.close(None).await?; + + Ok(()) +} + +pub struct RemoteDisplay { + id: Id, + rect: Rectangle, + children: Vec>, + pixmap: Pixmap, + socket_tx: tokio_mpsc::Sender, +} + +impl RemoteDisplay { + pub fn new( + rect: Rectangle, + hub: &Hub, + rq: &mut RenderQueue, + context: &mut Context, + ) -> RemoteDisplay { + let id = ID_FEEDER.next(); + let children = Vec::new(); + rq.add(RenderData::new(id, rect, UpdateMode::Full)); + + let tx = hub.clone(); + let my_tx = hub.clone(); + + let address = context.settings.remote_display.address.clone(); + let (socket_tx, socket_rx) = tokio_mpsc::channel(10); + spawn(move || { + let url = match Url::parse(&address) { + Ok(url) => url, + Err(e) => { + my_tx.send(Event::Back).ok(); + tx.send(Event::Notify(e.to_string())).ok(); + return; + } + }; + match display_connection(tx, socket_rx, url) { + Ok(_) => {} + Err(e) => { + my_tx.send(Event::Back).ok(); + my_tx.send(Event::Notify(e.to_string())).ok(); + } + } + }); + socket_tx + .try_send(SocketEvent::SendJSON(json!({ + "type": "size", + "value": { + "width": rect.width(), + "height": rect.height(), + } + }))) + .ok(); + + RemoteDisplay { + id, + rect, + children, + socket_tx, + pixmap: Pixmap::new(rect.width(), rect.height()), + } + } + + fn update_remote_view(&mut self, deflated_data: Box>) -> Result<(), Error> { + let mut inflated = Vec::new(); + ZlibDecoder::new(deflated_data.as_slice()).read_to_end(&mut inflated)?; + let dec = PnmDecoder::new(inflated.as_slice())?; + let (width, height) = dec.dimensions(); + let mut pixmap = Pixmap::new(width, height); + dec.read_image(&mut pixmap.data_mut())?; + self.pixmap = pixmap; + self.socket_tx + .try_send(SocketEvent::SendJSON(json!({ + "type": "displayUpdated", + }))) + .ok(); + Ok(()) + } +} + +impl View for RemoteDisplay { + fn handle_event( + &mut self, + evt: &Event, + hub: &Hub, + bus: &mut Bus, + rq: &mut RenderQueue, + _context: &mut Context, + ) -> bool { + match *evt { + Event::Gesture(GestureEvent::Arrow { + dir: Dir::South, + start: _start, + end: _end, + }) => { + self.socket_tx.try_send(SocketEvent::Finished).ok(); + bus.push_back(Event::Back); + true + } + Event::Gesture(ge) => { + self.socket_tx + .try_send(SocketEvent::SendJSON(serde_json::to_value(ge).unwrap())) + .ok(); + true + } + Event::UpdateRemoteView(ref png_data) => { + let data = png_data.clone(); + match self.update_remote_view(data) { + Ok(_) => {} + Err(e) => { + println!("{}", e); + hub.send(Event::Notify(e.to_string())).unwrap(); + } + } + rq.add(RenderData::new(self.id, self.rect, UpdateMode::Gui)); + true + } + Event::Update(UpdateMode::Full) => { + rq.add(RenderData::new(self.id, self.rect, UpdateMode::Full)); + true + } + _ => false, + } + } + + fn render(&self, fb: &mut dyn Framebuffer, rect: Rectangle, _fonts: &mut Fonts) { + fb.draw_framed_pixmap_halftone(&self.pixmap, &rect, rect.min); + } + + fn render_rect(&self, rect: &Rectangle) -> Rectangle { + rect.intersection(&self.rect).unwrap_or(self.rect) + } + + fn is_background(&self) -> bool { + true + } + + fn resize(&mut self, rect: Rectangle, hub: &Hub, rq: &mut RenderQueue, context: &mut Context) { + // Floating windows. + for i in 0..self.children.len() { + self.children[i].resize(rect, hub, rq, context); + } + + self.rect = rect; + rq.add(RenderData::new(self.id, self.rect, UpdateMode::Full)); + } + + fn rect(&self) -> &Rectangle { + &self.rect + } + + fn rect_mut(&mut self) -> &mut Rectangle { + &mut self.rect + } + + fn children(&self) -> &Vec> { + &self.children + } + + fn children_mut(&mut self) -> &mut Vec> { + &mut self.children + } + + fn id(&self) -> Id { + self.id + } +} diff --git a/crates/emulator/src/main.rs b/crates/emulator/src/main.rs index 6afa982f..60daa743 100644 --- a/crates/emulator/src/main.rs +++ b/crates/emulator/src/main.rs @@ -31,6 +31,7 @@ use plato_core::view::calculator::Calculator; use plato_core::view::sketch::Sketch; use plato_core::view::touch_events::TouchEvents; use plato_core::view::rotation_values::RotationValues; +use plato_core::view::remote_display::RemoteDisplay; use plato_core::view::common::{locate, locate_by_id, transfer_notifications, overlapping_rectangle}; use plato_core::view::common::{toggle_input_history_menu, toggle_keyboard_layout_menu}; use plato_core::helpers::{load_toml, save_toml}; @@ -439,6 +440,9 @@ fn main() -> Result<(), Error> { AppCmd::RotationValues => { Box::new(RotationValues::new(context.fb.rect(), &mut rq, &mut context)) }, + AppCmd::RemoteDisplay => { + Box::new(RemoteDisplay::new(context.fb.rect(), &tx, &mut rq, &mut context)) + } }; transfer_notifications(view.as_mut(), next_view.as_mut(), &mut rq, &mut context); history.push(view as Box); diff --git a/crates/plato/src/app.rs b/crates/plato/src/app.rs index 912464df..b84fbc40 100644 --- a/crates/plato/src/app.rs +++ b/crates/plato/src/app.rs @@ -20,6 +20,7 @@ use plato_core::view::calculator::Calculator; use plato_core::view::sketch::Sketch; use plato_core::view::touch_events::TouchEvents; use plato_core::view::rotation_values::RotationValues; +use plato_core::view::remote_display::RemoteDisplay; use plato_core::document::sys_info_as_html; use plato_core::input::{DeviceEvent, PowerSource, ButtonCode, ButtonStatus, VAL_RELEASE, VAL_PRESS}; use plato_core::input::{raw_events, device_events, usb_events, display_rotate_event, button_scheme_event}; @@ -815,6 +816,9 @@ pub fn run() -> Result<(), Error> { AppCmd::RotationValues => { Box::new(RotationValues::new(context.fb.rect(), &mut rq, &mut context)) }, + AppCmd::RemoteDisplay => { + Box::new(RemoteDisplay::new(context.fb.rect(), &tx, &mut rq, &mut context)) + } }; transfer_notifications(view.as_mut(), next_view.as_mut(), &mut rq, &mut context); history.push(HistoryItem { From 20b0e37a004b33cef0cfa0177e659fb2286ccd84 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Wed, 23 Aug 2023 05:58:27 -0500 Subject: [PATCH 02/79] Popup settings and persistency in remote display --- contrib/remote-display-webext/background.js | 251 +++++++++++--------- contrib/remote-display-webext/manifest.json | 3 + contrib/remote-display-webext/popup.html | 21 ++ contrib/remote-display-webext/popup.js | 22 ++ 4 files changed, 188 insertions(+), 109 deletions(-) create mode 100644 contrib/remote-display-webext/popup.html create mode 100644 contrib/remote-display-webext/popup.js diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index af9bff0a..836ba973 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -116,14 +116,6 @@ async function clickUnderTap(pctX, pctY) { // #endregion // #region main loop -async function getWsUrl() { - let wsUrl = "ws://localhost:8222/browser"; - const result = await browser.storage.local.get("wsUrl"); - if (result.wsUrl) { - wsUrl = result.wsUrl; - } - return wsUrl; -} let deviceWidth = 0; let deviceHeight = 0; @@ -167,114 +159,155 @@ browser.tabs.onUpdated.addListener(async (_id, changeInfo, tab) => { }, 1000); }); -async function connect() { - console.log("attempting connection"); - if (ws) ws.close(); - ws = new WebSocket(await getWsUrl()); - ws.onmessage = async (e) => { - console.log("message", e); - const msg = JSON.parse(e.data); - switch (msg.type) { - case "size": { - const { width, height } = msg.value; - deviceWidth = width; - deviceHeight = height; - await sendImage(); - break; - } - case "swipe": { - const { dir, start, end } = msg.value; - switch (dir) { - case "north": - case "south": { - const dy = end.y - start.y; - await scroll( - start.x / deviceWidth, - start.y / deviceHeight, - -(dy / deviceHeight), - ); - await sendImage(); - break; - } - case "east": - case "west": { - const offset = dir === "east" ? -1 : 1; - const dx = end.x - start.x; - // change windows if swipe covers more than half the screen - await (Math.abs(dx) > deviceHeight / 2 - ? windowOffset(offset) - : tabOffset(offset)); - await sendImage(); - const info = await currentTabInfo(); - sendNotice(info); - break; - } +async function onMessage(e) { + console.log("message", e); + const msg = JSON.parse(e.data); + switch (msg.type) { + case "size": { + const { width, height } = msg.value; + deviceWidth = width; + deviceHeight = height; + await sendImage(); + break; + } + case "swipe": { + const { dir, start, end } = msg.value; + switch (dir) { + case "north": + case "south": { + const dy = end.y - start.y; + await scroll( + start.x / deviceWidth, + start.y / deviceHeight, + -(dy / deviceHeight), + ); + await sendImage(); + break; } - break; - } - case "arrow": { - const { dir } = msg.value; - switch (dir) { - case "east": - await goForward(); - break; - case "west": - await goBack(); - break; - case "north": { - const info = await currentTabInfo(); - sendNotice(`closing ${info}`); - await closeCurrentTab(); - await sendImage(); - const newInfo = await currentTabInfo(); - sendNotice(newInfo); - break; - } + case "east": + case "west": { + const offset = dir === "east" ? -1 : 1; + const dx = end.x - start.x; + // change windows if swipe covers more than half the screen + await (Math.abs(dx) > deviceHeight / 2 + ? windowOffset(offset) + : tabOffset(offset)); + await sendImage(); + const info = await currentTabInfo(); + sendNotice(info); + break; } - break; - } - case "pinch": - await zoomPage(-0.1); - await sendImage(); - break; - case "spread": - await zoomPage(0.1); - await sendImage(); - break; - case "rotate": - await reloadCurrentTab(); - break; - case "holdFingerShort": - await resizeViewport(deviceWidth, deviceHeight); - await sendImage(); - await ws.send(JSON.stringify({ type: "refreshDisplay" })); - break; - case "holdFingerLong": { - const [{ x, y }] = msg.value; - const tab = await openLinkUnderTap(x / deviceWidth, y / deviceHeight); - if (tab) sendNotice(`${tab} opened`); - else sendNotice("no link under finger"); - break; } - case "tap": { - const { x, y } = msg.value; - await clickUnderTap(x / deviceWidth, y / deviceHeight); - await sendImage(); - break; + break; + } + case "arrow": { + const { dir } = msg.value; + switch (dir) { + case "east": + await goForward(); + break; + case "west": + await goBack(); + break; + case "north": { + const info = await currentTabInfo(); + sendNotice(`closing ${info}`); + await closeCurrentTab(); + await sendImage(); + const newInfo = await currentTabInfo(); + sendNotice(newInfo); + break; + } } + break; } - }; -} - -async function mainLoop() { - for (;;) { - if (!ws || ws.readyState === WebSocket.CLOSED) { - console.log("connection closed, restarting"); - await connect(); + case "pinch": + await zoomPage(-0.1); + await sendImage(); + break; + case "spread": + await zoomPage(0.1); + await sendImage(); + break; + case "rotate": + await reloadCurrentTab(); + break; + case "holdFingerShort": + await resizeViewport(deviceWidth, deviceHeight); + await sendImage(); + await ws.send(JSON.stringify({ type: "refreshDisplay" })); + break; + case "holdFingerLong": { + const [{ x, y }] = msg.value; + const tab = await openLinkUnderTap(x / deviceWidth, y / deviceHeight); + if (tab) sendNotice(`${tab} opened`); + else sendNotice("no link under finger"); + break; + } + case "tap": { + const { x, y } = msg.value; + await clickUnderTap(x / deviceWidth, y / deviceHeight); + await sendImage(); + break; } - await new Promise((resolve) => setTimeout(resolve, 5000)); } } -mainLoop().catch((e) => console.error(e)); +const defaultConfig = { + wsUrl: "ws://localhost:8222/browser", + enabled: false, +}; + +async function getConfig() { + const result = await browser.storage.local.get(["wsUrl", "enabled"]); + await browser.storage.local.set({ ...defaultConfig, ...result }); + return { ...defaultConfig, ...result }; +} + +const RETRY_TIMEOUT = 5000; +let { enabled, wsUrl } = defaultConfig; +function connect(config) { + enabled = config.enabled; + wsUrl = config.wsUrl; + if (!enabled) return; + ws = new WebSocket(wsUrl); + ws.addEventListener("open", () => { + console.log("connected"); + }); + ws.addEventListener("close", () => { + if (!enabled) return; + console.log("disconnected"); + setTimeout(() => { + getConfig().then(connect).catch(console.error); + }, RETRY_TIMEOUT); + }); + ws.addEventListener("message", onMessage); +} + +getConfig().then(connect).catch(console.error); + +browser.storage.onChanged.addListener((changes) => { + if ( + "enabled" in changes && + changes.enabled.newValue && + !changes.enabled.oldValue && + (!ws || ws.readyState === WebSocket.CLOSED) + ) { + getConfig().then(connect).catch(console.error); + } else if ( + "enabled" in changes && + !changes.enabled.newValue && + changes.enabled.oldValue && + ws + ) { + ws.close(); + } else if ( + "wsUrl" in changes && + changes.wsUrl.newValue !== changes.wsUrl.oldValue && + ws + ) { + ws.close(); + } +}); + // #endregion diff --git a/contrib/remote-display-webext/manifest.json b/contrib/remote-display-webext/manifest.json index 4f2f6918..08a4e90d 100644 --- a/contrib/remote-display-webext/manifest.json +++ b/contrib/remote-display-webext/manifest.json @@ -6,6 +6,9 @@ "background": { "scripts": ["background.js"] }, + "browser_action": { + "default_popup": "popup.html" + }, "permissions": [ "tabs", "", diff --git a/contrib/remote-display-webext/popup.html b/contrib/remote-display-webext/popup.html new file mode 100644 index 00000000..c0df1d9a --- /dev/null +++ b/contrib/remote-display-webext/popup.html @@ -0,0 +1,21 @@ + + + + Settings + + + +
+ + + + + + + +
+ + + + + \ No newline at end of file diff --git a/contrib/remote-display-webext/popup.js b/contrib/remote-display-webext/popup.js new file mode 100644 index 00000000..8282d0f0 --- /dev/null +++ b/contrib/remote-display-webext/popup.js @@ -0,0 +1,22 @@ +const form = document.getElementById('form'); +const urlInput = document.getElementById('url'); +const enabledInput = document.getElementById('enabled'); + +form.addEventListener('submit', async (e) => { + e.preventDefault(); + + await browser.storage.local.set({ + wsUrl: urlInput.value, + enabled: enabledInput.checked + }); + + // deno-lint-ignore no-window-prefix + window.close(); +}); + +(async () => { + const {wsUrl, enabled} = await browser.storage.local.get(['wsUrl', 'enabled']); + + urlInput.value = wsUrl || ''; + enabledInput.checked = enabled || false; +})(); From 71a65a2f37040ecdb06a69f3b08fc28a30685361 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Wed, 23 Aug 2023 05:59:58 -0500 Subject: [PATCH 03/79] Local storage --- contrib/remote-display-webext/background.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 836ba973..1bc05497 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -286,7 +286,7 @@ function connect(config) { getConfig().then(connect).catch(console.error); -browser.storage.onChanged.addListener((changes) => { +browser.storage.local.onChanged.addListener((changes) => { if ( "enabled" in changes && changes.enabled.newValue && From 3347a33033c9e0da8c9e69b1c2b8e41090f584ba Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Wed, 23 Aug 2023 08:00:20 -0500 Subject: [PATCH 04/79] Adjust contrast on remote tab with bottom corners --- contrib/remote-display-webext/background.js | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 1bc05497..0c37598d 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -113,6 +113,23 @@ async function clickUnderTap(pctX, pctY) { `document.elementFromPoint(window.innerWidth * ${pctX}, window.innerHeight * ${pctY})?.click()`, }); } + +async function offsetContrastFilter(offset) { + const [newContrast] = await browser.tabs.executeScript({ + code: `(() => { + const el = document.documentElement; + const match = el.style.filter?.match(/contrast\\((\\d+)%\\)/); + const contrast = match ? parseInt(match[1]) : 100; + const offsetContrast = contrast + ${offset}; + const newContrast = offsetContrast < 100 ? 100 : offsetContrast; + el.style.filter = newContrast !== 100 + ? "grayscale() contrast(" + newContrast + "%)" + : ""; + return newContrast; + })()`, + }); + return newContrast; +} // #endregion // #region main loop @@ -250,6 +267,22 @@ async function onMessage(e) { await sendImage(); break; } + case "corner": { + const { dir } = msg.value; + switch (dir) { + case "southWest": + case "southEast": + { + const newContrast = await offsetContrastFilter( + dir === "southWest" ? -25 : 25, + ); + sendNotice(`contrast ${newContrast}%`); + await sendImage(); + } + break; + } + break; + } } } From 427896ed6f237c765b554ef7da17e13f30057388 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Fri, 25 Aug 2023 05:05:16 -0500 Subject: [PATCH 05/79] Top corners to rescale window, better than zoom --- contrib/remote-display-webext/background.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 0c37598d..3256b7a3 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -74,6 +74,8 @@ async function reloadCurrentTab() { } async function resizeViewport(width, height) { + width = Math.round(width); + height = Math.round(height); const window = await browser.windows.getCurrent(); const tab = await currentTab(); if (tab.width === width && tab.height === height) return; @@ -134,6 +136,7 @@ async function offsetContrastFilter(offset) { // #region main loop +let scaleFactor = 1; let deviceWidth = 0; let deviceHeight = 0; let ws; @@ -141,7 +144,9 @@ let ws; async function sendImage() { if (!ws || ws.readyState !== WebSocket.OPEN) return; console.log("capturing"); - const dataUrl = await browser.tabs.captureVisibleTab(); + const dataUrl = await browser.tabs.captureVisibleTab(undefined, { + scale: scaleFactor, + }); const buf = await (await fetch(dataUrl)).arrayBuffer(); ws.send(buf); await new Promise((resolve) => { @@ -250,7 +255,7 @@ async function onMessage(e) { await reloadCurrentTab(); break; case "holdFingerShort": - await resizeViewport(deviceWidth, deviceHeight); + await resizeViewport(deviceWidth / scaleFactor, deviceHeight / scaleFactor); await sendImage(); await ws.send(JSON.stringify({ type: "refreshDisplay" })); break; @@ -280,6 +285,15 @@ async function onMessage(e) { await sendImage(); } break; + case "northWest": + case "northEast": { + const newScaleFactor = scaleFactor + (dir === "northWest" ? -0.1 : 0.1); + if (newScaleFactor < 0.1 || newScaleFactor > 2) break; + scaleFactor = newScaleFactor; + sendNotice(`scale ${Math.round(scaleFactor * 100)}%`); + await resizeViewport(deviceWidth / scaleFactor, deviceHeight / scaleFactor); + await sendImage(); + } } break; } From 9478dc367ce72ff542276b1c340fd1cf39302fd2 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Fri, 8 Sep 2023 09:55:32 -0500 Subject: [PATCH 06/79] Toggle invert at lowest contrast setting --- contrib/remote-display-webext/background.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 3256b7a3..320dbc19 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -122,12 +122,14 @@ async function offsetContrastFilter(offset) { const el = document.documentElement; const match = el.style.filter?.match(/contrast\\((\\d+)%\\)/); const contrast = match ? parseInt(match[1]) : 100; + const invert = el.style.filter?.includes("invert"); const offsetContrast = contrast + ${offset}; - const newContrast = offsetContrast < 100 ? 100 : offsetContrast; - el.style.filter = newContrast !== 100 - ? "grayscale() contrast(" + newContrast + "%)" - : ""; - return newContrast; + if (offsetContrast < 100) { + el.style.filter = \`grayscale() \${invert ? "" : "invert() "}contrast(100%)\`; + } else { + el.style.filter = \`grayscale() \${invert ? "invert() " : ""}contrast(\${offsetContrast}%)\`; + } + return el.style.filter.match(/contrast\\((\\d+)%\\)/)[1]; })()`, }); return newContrast; From a4834ca34340ebfe8fd0cb9d3b53668ac1e16118 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Fri, 8 Sep 2023 14:01:37 -0500 Subject: [PATCH 07/79] Ignore bugged rotate gesture events --- contrib/remote-display-webext/background.js | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 320dbc19..d9eb402b 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -254,6 +254,7 @@ async function onMessage(e) { await sendImage(); break; case "rotate": + if (Math.abs(msg.value.angle) < 20) break; await reloadCurrentTab(); break; case "holdFingerShort": From 306b7fd5be5bcc916f60223cdf3f21defbc02b14 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:36:49 -0500 Subject: [PATCH 08/79] Remove grayscale at normal contrast always --- contrib/remote-display-webext/background.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index d9eb402b..c9012f63 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -125,9 +125,12 @@ async function offsetContrastFilter(offset) { const invert = el.style.filter?.includes("invert"); const offsetContrast = contrast + ${offset}; if (offsetContrast < 100) { - el.style.filter = \`grayscale() \${invert ? "" : "invert() "}contrast(100%)\`; + el.style.filter = \`\${invert ? "" : "grayscale() invert() "}contrast(100%)\`; } else { - el.style.filter = \`grayscale() \${invert ? "invert() " : ""}contrast(\${offsetContrast}%)\`; + el.style.filter = \`\${ + (offsetContrast === 100) && !invert ? "" : "grayscale() "}\${ + invert ? "invert() " : "" + }contrast(\${offsetContrast}%)\`; } return el.style.filter.match(/contrast\\((\\d+)%\\)/)[1]; })()`, From 8fbf5a498325d90047879f04751620fa441ae30c Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Mon, 25 Sep 2023 12:56:32 -0500 Subject: [PATCH 09/79] Get cargo lock from upstream --- Cargo.lock | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 10bd493a..7c2b7675 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -150,9 +150,9 @@ checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "bytemuck" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" [[package]] name = "byteorder" @@ -282,15 +282,17 @@ dependencies = [ ] [[package]] -name = "deranged" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" name = "data-encoding" version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +[[package]] +name = "deranged" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" + [[package]] name = "digest" version = "0.10.7" @@ -931,9 +933,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project-lite" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -970,7 +972,7 @@ dependencies = [ "fxhash", "globset", "image", - "indexmap 1.9.3", + "indexmap 2.0.0", "kl-hyphenate", "lazy_static", "levenshtein", @@ -1376,9 +1378,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys", @@ -1510,9 +1512,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2dbec703c26b00d74844519606ef15d09a7d6857860f84ad223dec002ddea2" +checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", @@ -1602,9 +1604,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "tungstenite" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e862a1c4128df0112ab625f55cd5c934bcb4312ba80b39ae4b4835a3fd58e649" +checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" dependencies = [ "byteorder", "bytes", From 73e69d28ca4a0d1afba04f49f9202f429a277701 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Mon, 25 Sep 2023 13:00:20 -0500 Subject: [PATCH 10/79] PNG -> PBM --- crates/core/src/view/remote_display/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/core/src/view/remote_display/mod.rs b/crates/core/src/view/remote_display/mod.rs index 25fc9699..33abf999 100644 --- a/crates/core/src/view/remote_display/mod.rs +++ b/crates/core/src/view/remote_display/mod.rs @@ -194,10 +194,10 @@ impl View for RemoteDisplay { .ok(); true } - Event::UpdateRemoteView(ref png_data) => { - let data = png_data.clone(); + Event::UpdateRemoteView(ref pbm_data) => { + let data = pbm_data.clone(); match self.update_remote_view(data) { - Ok(_) => {} + Ok(..) => {} Err(e) => { println!("{}", e); hub.send(Event::Notify(e.to_string())).unwrap(); From 9aa03f8e034802b18c1ac518b1f61e9639c3bc86 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:13:24 -0500 Subject: [PATCH 11/79] Device button presses to scroll half page --- contrib/remote-display-webext/background.js | 22 +++++++++++++++++--- crates/core/src/view/remote_display/mod.rs | 23 +++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index c9012f63..7cd6c2b8 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -29,9 +29,8 @@ async function currentTabInfo() { const currentWindow = windows.find((window) => window.focused); const currentWindowIndex = windows.indexOf(currentWindow); const url = new URL(currentTab.url); - return `W${currentWindowIndex + 1} T${ - currentTabIndex + 1 - }/${tabs.length} ${url.host}`; + return `W${currentWindowIndex + 1} T${currentTabIndex + 1 + }/${tabs.length} ${url.host}`; } async function scroll(pctX, pctY, pct) { @@ -227,6 +226,23 @@ async function onMessage(e) { } break; } + case "button": { + const { button, status } = msg.value; + if (status !== "released") break; + // scroll half pages + switch (button) { + case "forward": { + await scroll(0.5, 0.5, 0.5); + break; + } + case "backward": { + await scroll(0.5, 0.5, -0.5); + break; + } + } + await sendImage(); + break; + } case "arrow": { const { dir } = msg.value; switch (dir) { diff --git a/crates/core/src/view/remote_display/mod.rs b/crates/core/src/view/remote_display/mod.rs index 33abf999..93a6036c 100644 --- a/crates/core/src/view/remote_display/mod.rs +++ b/crates/core/src/view/remote_display/mod.rs @@ -3,6 +3,7 @@ use crate::font::Fonts; use crate::framebuffer::{Framebuffer, Pixmap, UpdateMode}; use crate::geom::{Dir, Rectangle}; use crate::gesture::GestureEvent; +use crate::input::{ButtonCode, ButtonStatus, DeviceEvent}; use crate::view::{Bus, Event, Hub, RenderData, RenderQueue, View}; use crate::view::{Id, ID_FEEDER}; use anyhow::Error; @@ -194,6 +195,28 @@ impl View for RemoteDisplay { .ok(); true } + Event::Device(DeviceEvent::Button { code, status, .. }) => { + let button = match code { + ButtonCode::Forward => "forward", + ButtonCode::Backward => "backward", + _ => return false, + }; + let status = match status { + ButtonStatus::Pressed => "pressed", + ButtonStatus::Released => "released", + ButtonStatus::Repeated => "repeated", + }; + self.socket_tx + .try_send(SocketEvent::SendJSON(json!({ + "type": "button", + "value": { + "button": button, + "status": status, + } + }))) + .ok(); + true + } Event::UpdateRemoteView(ref pbm_data) => { let data = pbm_data.clone(); match self.update_remote_view(data) { From e93ece70427c9ef2addc036011b55b3fada9707e Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Sun, 8 Oct 2023 00:41:10 -0500 Subject: [PATCH 12/79] Decouple display from currently focused window --- contrib/remote-display-webext/background.js | 58 +++++++++++++-------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 7cd6c2b8..46850398 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -1,6 +1,13 @@ // #region browser interactivity + +let windowId; +browser.tabs.query({ active: true, currentWindow: true }) + .then((tabs) => { + windowId = tabs[0].windowId; + }); + async function tabOffset(offset) { - const tabs = await browser.tabs.query({ currentWindow: true }); + const tabs = await browser.tabs.query({ windowId }); const currentTab = tabs.find((tab) => tab.active); const currentIndex = tabs.indexOf(currentTab); const newIndex = (currentIndex + offset + tabs.length) % tabs.length; @@ -9,24 +16,24 @@ async function tabOffset(offset) { async function windowOffset(offset) { const windows = await browser.windows.getAll({ populate: true }); - const currentWindow = windows.find((window) => window.focused); + const currentWindow = windows.find((window) => window.id === windowId); const currentIndex = windows.indexOf(currentWindow); const newIndex = (currentIndex + offset + windows.length) % windows.length; - await browser.windows.update(windows[newIndex].id, { focused: true }); + windowId = windows[newIndex].id; await new Promise((resolve) => setTimeout(resolve, 100)); } async function currentTab() { - const tabs = await browser.tabs.query({ currentWindow: true }); - return tabs.find((tab) => tab.active); + const [tab] = await browser.tabs.query({ windowId, active: true }); + return tab; } async function currentTabInfo() { - const tabs = await browser.tabs.query({ currentWindow: true }); + const tabs = await browser.tabs.query({ windowId }); const currentTab = tabs.find((tab) => tab.active); const currentTabIndex = tabs.indexOf(currentTab); const windows = await browser.windows.getAll({ populate: true }); - const currentWindow = windows.find((window) => window.focused); + const currentWindow = windows.find((window) => window.id === windowId); const currentWindowIndex = windows.indexOf(currentWindow); const url = new URL(currentTab.url); return `W${currentWindowIndex + 1} T${currentTabIndex + 1 @@ -34,7 +41,8 @@ async function currentTabInfo() { } async function scroll(pctX, pctY, pct) { - await browser.tabs.executeScript({ + const { id } = await currentTab(); + await browser.tabs.executeScript(id, { code: `(() => { const el = [...document.elementsFromPoint( window.innerWidth * ${pctX}, window.innerHeight * ${pctY} @@ -49,33 +57,38 @@ async function scroll(pctX, pctY, pct) { } async function zoomPage(addFactor) { - const factor = await browser.tabs.getZoom(); + const { id } = await currentTab(); + const factor = await browser.tabs.getZoom(id); let newFactor = factor + addFactor; if (newFactor < 0.3) newFactor = 0.3; if (newFactor > 5) newFactor = 5; - await browser.tabs.setZoom(undefined, newFactor); + await browser.tabs.setZoom(id, newFactor); } async function goForward() { - await browser.tabs.goForward(); + const { id } = await currentTab(); + await browser.tabs.goForward(id); } async function goBack() { - await browser.tabs.goBack(); + const { id } = await currentTab(); + await browser.tabs.goBack(id); } async function closeCurrentTab() { - await browser.tabs.remove((await currentTab()).id); + const { id } = await currentTab(); + await browser.tabs.remove(id); } async function reloadCurrentTab() { - await browser.tabs.reload(); + const { id } = await currentTab(); + await browser.tabs.reload(id); } async function resizeViewport(width, height) { width = Math.round(width); height = Math.round(height); - const window = await browser.windows.getCurrent(); + const window = await browser.windows.get(windowId); const tab = await currentTab(); if (tab.width === width && tab.height === height) return; if (tab.width > width || tab.height > height) { @@ -95,8 +108,8 @@ async function resizeViewport(width, height) { } async function openLinkUnderTap(pctX, pctY) { - const tab = await currentTab(); - const [url] = await browser.tabs.executeScript({ + const { id } = await currentTab(); + const [url] = await browser.tabs.executeScript(id, { code: `[...document.elementsFromPoint(window.innerWidth * ${pctX}, window.innerHeight * ${pctY})] .find((e) => !!e.href) @@ -104,19 +117,21 @@ async function openLinkUnderTap(pctX, pctY) { }); if (!url) return; await browser.tabs.create({ url }); - await browser.tabs.update(tab.id, { active: true }); + await browser.tabs.update(id, { active: true }); return new URL(url).host; } async function clickUnderTap(pctX, pctY) { - await browser.tabs.executeScript({ + const { id } = await currentTab(); + await browser.tabs.executeScript(id, { code: `document.elementFromPoint(window.innerWidth * ${pctX}, window.innerHeight * ${pctY})?.click()`, }); } async function offsetContrastFilter(offset) { - const [newContrast] = await browser.tabs.executeScript({ + const { id } = await currentTab(); + const [newContrast] = await browser.tabs.executeScript(id, { code: `(() => { const el = document.documentElement; const match = el.style.filter?.match(/contrast\\((\\d+)%\\)/); @@ -148,7 +163,8 @@ let ws; async function sendImage() { if (!ws || ws.readyState !== WebSocket.OPEN) return; console.log("capturing"); - const dataUrl = await browser.tabs.captureVisibleTab(undefined, { + const { id } = await currentTab(); + const dataUrl = await browser.tabs.captureTab(id, { scale: scaleFactor, }); const buf = await (await fetch(dataUrl)).arrayBuffer(); From 2cd6ba952cea1de1988c2171d3c8e757ee80a0fc Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Wed, 11 Oct 2023 18:51:13 -0500 Subject: [PATCH 13/79] Remove redundant viewport resize --- contrib/remote-display-webext/background.js | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 46850398..4fad3960 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -96,7 +96,6 @@ async function resizeViewport(width, height) { width: width, height: height, }); - await resizeViewport(width, height); } const offsetWidth = window.width - tab.width; const offsetHeight = window.height - tab.height; From b7bb4b7017d34f48541f92f20d10e4a7a461fe67 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Wed, 11 Oct 2023 18:56:29 -0500 Subject: [PATCH 14/79] Handle basic authentication and TLS --- Cargo.lock | 5 +++++ contrib/remote-display-server.ts | 19 +++++++++++++++++- crates/core/Cargo.toml | 3 ++- crates/core/src/view/remote_display/mod.rs | 23 +++++++++++++++++++++- 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c2b7675..828b4124 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,6 +961,7 @@ name = "plato-core" version = "0.9.39" dependencies = [ "anyhow", + "base64", "bitflags 2.4.0", "byteorder", "chrono", @@ -1518,8 +1519,11 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", + "rustls", "tokio", + "tokio-rustls", "tungstenite", + "webpki-roots", ] [[package]] @@ -1615,6 +1619,7 @@ dependencies = [ "httparse", "log", "rand", + "rustls", "sha1", "thiserror", "url", diff --git a/contrib/remote-display-server.ts b/contrib/remote-display-server.ts index 75f47f8a..5c9b3df4 100644 --- a/contrib/remote-display-server.ts +++ b/contrib/remote-display-server.ts @@ -36,7 +36,6 @@ async function convertToBlob( // #endregion const app = new Application(); -const port = parseInt(Deno.env.get("PORT") || "8222"); const router = new Router(); let deviceSocket: WebSocket | undefined; @@ -124,8 +123,26 @@ router.get("/device", (ctx) => { }); // #endregion +const password = Deno.env.get("REMOTE_DISPLAY_PASSWORD"); +if (password) { + app.use(async (ctx, next) => { + const auth = ctx.request.headers.get("authorization"); + const expectedAuth = `Basic ${btoa(`plato:${password}`)}`; + if (auth !== expectedAuth) { + console.log(`Unauthorized access from ${ctx.request.ip}`); + ctx.response.status = 401; + ctx.response.headers.set("WWW-Authenticate", "Basic"); + ctx.response.body = "Unauthorized"; + return; + } + await next(); + }); + console.log("Password protected with username 'plato'"); +} + app.use(router.routes()); app.use(router.allowedMethods()); +const port = parseInt(Deno.env.get("PORT") || "8222"); console.log("Listening at port " + port); await app.listen({ port }); diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 819e7d31..ec1ce317 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -42,6 +42,7 @@ percent-encoding = "2.3.0" chrono = { version = "0.4.30", features = ["serde", "clock"], default-features = false } image = { version = "0.24.7", features = ["pnm"], default-features = false } url = { version = "2.1.0" } -tokio-tungstenite = "0.20.0" +base64 = "0.21.4" +tokio-tungstenite = { version = "0.20.0", features = ["rustls-tls-webpki-roots"] } tokio = { version = "1.32.0", features = ["macros", "rt", "net", "sync"] } futures-util = { version = "0.3.28", default-features = false, features = ["sink", "std"] } diff --git a/crates/core/src/view/remote_display/mod.rs b/crates/core/src/view/remote_display/mod.rs index 93a6036c..d464d0c2 100644 --- a/crates/core/src/view/remote_display/mod.rs +++ b/crates/core/src/view/remote_display/mod.rs @@ -7,6 +7,7 @@ use crate::input::{ButtonCode, ButtonStatus, DeviceEvent}; use crate::view::{Bus, Event, Hub, RenderData, RenderQueue, View}; use crate::view::{Id, ID_FEEDER}; use anyhow::Error; +use base64::{engine::general_purpose, Engine as _}; use flate2::bufread::ZlibDecoder; use futures_util::{SinkExt, StreamExt}; use image::codecs::pnm::PnmDecoder; @@ -17,6 +18,7 @@ use std::io::Read; use std::sync::mpsc as std_mpsc; use std::thread::spawn; use tokio::sync::mpsc as tokio_mpsc; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; use tokio_tungstenite::{connect_async, tungstenite::protocol::Message}; use url::Url; @@ -39,7 +41,26 @@ async fn display_connection( mut socket_rx: tokio_mpsc::Receiver, url: Url, ) -> Result<(), Box> { - let (mut socket, _) = connect_async(url).await?; + let username = url.username(); + let password = url.password().unwrap_or(""); + + let mut request = url.clone().into_client_request()?; + match username { + "" => {} + username => { + request.headers_mut().append( + "Authorization", + format!( + "Basic {}", + general_purpose::STANDARD + .encode(format!("{}:{}", username, password).as_bytes()) + ) + .parse()?, + ); + } + } + + let (mut socket, _) = connect_async(request).await?; event_tx.send(Event::Notify("Connected".to_string()))?; loop { tokio::select! { From 5f115e019243a2cddf2bb3970ed2e9708ad2266b Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Thu, 12 Oct 2023 22:02:15 -0500 Subject: [PATCH 15/79] Rotate counterclockwise to reopen closed tab --- contrib/remote-display-webext/background.js | 19 ++++++++++++++++++- contrib/remote-display-webext/manifest.json | 3 ++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 4fad3960..4af27407 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -80,6 +80,16 @@ async function closeCurrentTab() { await browser.tabs.remove(id); } +async function reopenClosedTab() { + const sessions = await browser.sessions.getRecentlyClosed(); + const lastSession = sessions + .find((session) => session.tab && session.tab.windowId === windowId); + if (!lastSession) return; + await browser.sessions.restore(lastSession.tab.sessionId); + await browser.tabs.update(lastSession.tab.id, { active: true }); + return new URL(lastSession.tab.url).host; +} + async function reloadCurrentTab() { const { id } = await currentTab(); await browser.tabs.reload(id); @@ -289,7 +299,14 @@ async function onMessage(e) { break; case "rotate": if (Math.abs(msg.value.angle) < 20) break; - await reloadCurrentTab(); + if (msg.value.angle > 0) { + const restoredTab = await reopenClosedTab(); + if (restoredTab) sendNotice(`restored ${restoredTab}`); + else sendNotice("no tab restored"); + await sendImage(); + } else { + await reloadCurrentTab(); + } break; case "holdFingerShort": await resizeViewport(deviceWidth / scaleFactor, deviceHeight / scaleFactor); diff --git a/contrib/remote-display-webext/manifest.json b/contrib/remote-display-webext/manifest.json index 08a4e90d..994a37df 100644 --- a/contrib/remote-display-webext/manifest.json +++ b/contrib/remote-display-webext/manifest.json @@ -12,6 +12,7 @@ "permissions": [ "tabs", "", - "storage" + "storage", + "sessions" ] } From 0c6f9d247e9f752c7e80ed894cd60a31edd9b355 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Fri, 13 Oct 2023 04:02:48 -0500 Subject: [PATCH 16/79] Handle rotation during remote display session --- contrib/remote-display-server.ts | 7 +------ crates/core/src/view/remote_display/mod.rs | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/contrib/remote-display-server.ts b/contrib/remote-display-server.ts index 5c9b3df4..4f6df553 100644 --- a/contrib/remote-display-server.ts +++ b/contrib/remote-display-server.ts @@ -110,12 +110,7 @@ router.get("/device", (ctx) => { case "size": deviceWidth = msg.value.width; deviceHeight = msg.value.height; - $`convert -size ${deviceWidth}x${deviceHeight} xc:white -gravity center -font DejaVu-Sans-Mono-Book -pointsize 32 -annotate +0+0 "Welcome" pnm:-` - .bytes() - .then((data) => convertToBlob(data, deviceWidth, deviceHeight)) - .then((blob) => deviceSocket?.send(blob)); - browserSocket?.send(m.data); - break; + /* falls through */ default: browserSocket?.send(m.data); } diff --git a/crates/core/src/view/remote_display/mod.rs b/crates/core/src/view/remote_display/mod.rs index d464d0c2..9b9375a4 100644 --- a/crates/core/src/view/remote_display/mod.rs +++ b/crates/core/src/view/remote_display/mod.rs @@ -259,7 +259,17 @@ impl View for RemoteDisplay { } fn render(&self, fb: &mut dyn Framebuffer, rect: Rectangle, _fonts: &mut Fonts) { - fb.draw_framed_pixmap_halftone(&self.pixmap, &rect, rect.min); + let max_pixel_x = (rect.max.x - 1) as u32; + let max_pixel_y = (rect.max.y - 1) as u32; + let addr = (max_pixel_y * self.pixmap.width + max_pixel_x) as usize; + let out_of_bounds = addr >= self.pixmap.data.len(); + if out_of_bounds { + // Blank to prevent a panic when the pixmap is not the expected size. + let pixmap = Pixmap::new(fb.width(), fb.height()); + fb.draw_framed_pixmap(&pixmap, &rect, rect.min); + return; + } + fb.draw_framed_pixmap(&self.pixmap, &rect, rect.min); } fn render_rect(&self, rect: &Rectangle) -> Rectangle { @@ -276,6 +286,16 @@ impl View for RemoteDisplay { self.children[i].resize(rect, hub, rq, context); } + self.socket_tx + .try_send(SocketEvent::SendJSON(json!({ + "type": "size", + "value": { + "width": rect.width(), + "height": rect.height(), + } + }))) + .ok(); + self.rect = rect; rq.add(RenderData::new(self.id, self.rect, UpdateMode::Full)); } From dd6a4270fdfac798f5f3821fb586558c537732c6 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Fri, 15 Dec 2023 13:07:27 -0600 Subject: [PATCH 17/79] Merge cargo lock --- Cargo.lock | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e7ac998..48a82a8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -448,6 +448,7 @@ checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-core", "futures-io", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -945,6 +946,7 @@ name = "plato-core" version = "0.9.40" dependencies = [ "anyhow", + "base64", "bitflags 2.4.1", "byteorder", "chrono", @@ -955,6 +957,7 @@ dependencies = [ "futures-util", "fxhash", "globset", + "image", "indexmap", "kl-hyphenate", "lazy_static", @@ -1001,6 +1004,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "proc-macro2" version = "1.0.70" @@ -1490,14 +1499,15 @@ dependencies = [ "num_cpus", "pin-project-lite", "socket2 0.5.5", + "tokio-macros", "windows-sys", ] [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", From 54aaa7874d2f942180ec0ec09717b5359ca823cc Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Thu, 21 Dec 2023 03:56:31 -0600 Subject: [PATCH 18/79] Handle new Deno binary WebSocket messages --- contrib/remote-display-server.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/contrib/remote-display-server.ts b/contrib/remote-display-server.ts index 4f6df553..94a409be 100644 --- a/contrib/remote-display-server.ts +++ b/contrib/remote-display-server.ts @@ -69,11 +69,14 @@ router.get("/browser", (ctx) => { console.log("Browser disconnected"); }; - browserSocket.onmessage = (m) => { - if (m.data instanceof ArrayBuffer) { + browserSocket.onmessage = async (m) => { + if (m.data instanceof ArrayBuffer || m.data instanceof Blob) { console.log("Browser sends image"); if (!deviceSocket) return; - convertToBlob(new Uint8Array(m.data), deviceWidth, deviceHeight) + const data = m.data instanceof ArrayBuffer + ? m.data + : await m.data.arrayBuffer(); + convertToBlob(new Uint8Array(data), deviceWidth, deviceHeight) .then((blob) => deviceSocket?.send(blob)); return; } From a3ded985ed1e44d0a00230fd61d458a3b8e4b290 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Sun, 24 Dec 2023 01:16:41 -0600 Subject: [PATCH 19/79] Repeat scroll while back/forward button held --- contrib/remote-display-webext/background.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 4af27407..d0966ea1 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -253,7 +253,7 @@ async function onMessage(e) { } case "button": { const { button, status } = msg.value; - if (status !== "released") break; + if (!["released", "repeated"].includes(status)) break; // scroll half pages switch (button) { case "forward": { @@ -265,7 +265,7 @@ async function onMessage(e) { break; } } - await sendImage(); + if (status === "released") await sendImage(); break; } case "arrow": { From f31db8ea04716dda214231fed1e8fa4d3fc7eb52 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Thu, 4 Jan 2024 06:37:11 -0600 Subject: [PATCH 20/79] Handle errors in sending to closed sockets --- contrib/remote-display-server.ts | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/contrib/remote-display-server.ts b/contrib/remote-display-server.ts index 94a409be..10d662b1 100644 --- a/contrib/remote-display-server.ts +++ b/contrib/remote-display-server.ts @@ -76,12 +76,26 @@ router.get("/browser", (ctx) => { const data = m.data instanceof ArrayBuffer ? m.data : await m.data.arrayBuffer(); - convertToBlob(new Uint8Array(data), deviceWidth, deviceHeight) - .then((blob) => deviceSocket?.send(blob)); + const blob = await convertToBlob( + new Uint8Array(data), + deviceWidth, + deviceHeight, + ); + try { + deviceSocket.send(blob); + } catch (e) { + console.error("Error sending to device: ", e); + deviceSocket = undefined; + } return; } console.log("Browser sends: ", m.data); - deviceSocket?.send(m.data); + try { + deviceSocket?.send(m.data); + } catch (e) { + console.error("Error sending to device: ", e); + deviceSocket = undefined; + } }; }); // #endregion @@ -115,7 +129,12 @@ router.get("/device", (ctx) => { deviceHeight = msg.value.height; /* falls through */ default: - browserSocket?.send(m.data); + try { + browserSocket?.send(m.data); + } catch (e) { + console.error("Error sending to browser: ", e); + browserSocket = undefined; + } } }; }); @@ -127,7 +146,7 @@ if (password) { const auth = ctx.request.headers.get("authorization"); const expectedAuth = `Basic ${btoa(`plato:${password}`)}`; if (auth !== expectedAuth) { - console.log(`Unauthorized access from ${ctx.request.ip}`); + console.error(`Unauthorized access from ${ctx.request.ip}`); ctx.response.status = 401; ctx.response.headers.set("WWW-Authenticate", "Basic"); ctx.response.body = "Unauthorized"; From e4c4cb8543a59c8730dd99fe90374b87d06a3ac7 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Fri, 2 Feb 2024 06:21:58 -0600 Subject: [PATCH 21/79] Upgrade remote display server dependencies --- contrib/remote-display-server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/remote-display-server.ts b/contrib/remote-display-server.ts index 10d662b1..fe1c63f6 100644 --- a/contrib/remote-display-server.ts +++ b/contrib/remote-display-server.ts @@ -1,6 +1,6 @@ -import { Application, Router } from "https://deno.land/x/oak@v12.6.0/mod.ts"; +import { Application, Router } from "https://deno.land/x/oak@v13.0.0/mod.ts"; import { Foras, zlib } from "https://deno.land/x/foras@2.0.8/src/deno/mod.ts"; -import $ from "https://deno.land/x/dax@0.34.0/mod.ts"; +import $ from "https://deno.land/x/dax@0.38.0/mod.ts"; await Foras.initBundledOnce(); From 335199af55768d969f03abd0d9386910c570b741 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Thu, 14 Mar 2024 02:16:19 -0400 Subject: [PATCH 22/79] Close previous browser connection if it exists --- contrib/remote-display-server.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/contrib/remote-display-server.ts b/contrib/remote-display-server.ts index fe1c63f6..d4d6f01a 100644 --- a/contrib/remote-display-server.ts +++ b/contrib/remote-display-server.ts @@ -46,9 +46,8 @@ let deviceHeight = 0; // #region browser socket router.get("/browser", (ctx) => { if (browserSocket) { - ctx.response.status = 400; - ctx.response.body = "Only one browser connection allowed"; - return; + browserSocket.close(); + console.log("Closing previous browser connection"); } browserSocket = ctx.upgrade(); console.log("Browser starting connection"); From f6de53dd4e90e41bef66aa98aeb0326e29ba0d9c Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Thu, 14 Mar 2024 20:29:02 -0400 Subject: [PATCH 23/79] Move image conversion to webextension --- contrib/remote-display-server.ts | 58 ++------------------- contrib/remote-display-webext/background.js | 37 ++++++++++++- contrib/remote-display-webext/manifest.json | 4 +- 3 files changed, 42 insertions(+), 57 deletions(-) diff --git a/contrib/remote-display-server.ts b/contrib/remote-display-server.ts index d4d6f01a..dd1f1a04 100644 --- a/contrib/remote-display-server.ts +++ b/contrib/remote-display-server.ts @@ -1,39 +1,4 @@ import { Application, Router } from "https://deno.land/x/oak@v13.0.0/mod.ts"; -import { Foras, zlib } from "https://deno.land/x/foras@2.0.8/src/deno/mod.ts"; -import $ from "https://deno.land/x/dax@0.38.0/mod.ts"; - -await Foras.initBundledOnce(); - -// #region support functions -const bytes = Intl.NumberFormat("en", { - notation: "compact", - style: "unit", - unit: "byte", - unitDisplay: "narrow", -}); - -/** Use ImageMagick `convert` to create a deflated PBM. */ -async function convertToBlob( - data: Uint8Array, - width: number, - height: number, -): Promise { - const start = performance.now(); - const image = - await $`convert - -resize ${width}x${height}! -dither FloydSteinberg -remap pattern:gray50 pnm:-` - .stdin(data) - .bytes(); - const compressed = zlib(image); - console.log( - `converted ${bytes.format(data.byteLength)} frame to ${ - bytes.format(compressed.byteLength) - } (${bytes.format(image.byteLength)} inflated) in ${ - Math.round(performance.now() - start) - }ms`, - ); - return new Blob([compressed], { type: "application/x-deflate" }); -} -// #endregion const app = new Application(); const router = new Router(); @@ -68,27 +33,12 @@ router.get("/browser", (ctx) => { console.log("Browser disconnected"); }; - browserSocket.onmessage = async (m) => { + browserSocket.onmessage = (m) => { if (m.data instanceof ArrayBuffer || m.data instanceof Blob) { console.log("Browser sends image"); - if (!deviceSocket) return; - const data = m.data instanceof ArrayBuffer - ? m.data - : await m.data.arrayBuffer(); - const blob = await convertToBlob( - new Uint8Array(data), - deviceWidth, - deviceHeight, - ); - try { - deviceSocket.send(blob); - } catch (e) { - console.error("Error sending to device: ", e); - deviceSocket = undefined; - } - return; + } else { + console.log(`Browser sends ${m.data}`); } - console.log("Browser sends: ", m.data); try { deviceSocket?.send(m.data); } catch (e) { @@ -120,7 +70,7 @@ router.get("/device", (ctx) => { deviceSocket.onmessage = (m) => { if (m.data instanceof ArrayBuffer) return; - console.log("Device sends: ", m.data); + console.log(`Device sends ${m.data}`); const msg = JSON.parse(m.data); switch (msg.type) { case "size": diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index d0966ea1..8e6aab7c 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -169,6 +169,23 @@ let deviceWidth = 0; let deviceHeight = 0; let ws; +import { + ColorSpace, + DitherMethod, + ImageMagick, + initializeImageMagick, + MagickFormat, + QuantizeSettings, + MagickGeometry +} from "https://esm.sh/@imagemagick/magick-wasm@0.0.28"; +import { Foras, Memory, zlib } from "https://esm.sh/@hazae41/foras@2.1.4"; +const wasmFile = + "https://esm.sh/@imagemagick/magick-wasm@0.0.28/dist/magick.wasm"; +await fetch(wasmFile, { cache: "force-cache" }) + .then(a => a.arrayBuffer()) + .then(a => initializeImageMagick(a)); +await Foras.initBundledOnce(); + async function sendImage() { if (!ws || ws.readyState !== WebSocket.OPEN) return; console.log("capturing"); @@ -176,8 +193,24 @@ async function sendImage() { const dataUrl = await browser.tabs.captureTab(id, { scale: scaleFactor, }); - const buf = await (await fetch(dataUrl)).arrayBuffer(); - ws.send(buf); + const buf = await fetch(dataUrl) + .then(a => a.arrayBuffer()) + .then(a => new Uint8Array(a)); + + ImageMagick.read(buf, (img) => { + const mg = new MagickGeometry(deviceWidth, deviceHeight); + mg.ignoreAspectRatio = true; + const qs = new QuantizeSettings(); + qs.colors = 2; + qs.colorSpace = ColorSpace.Gray; + qs.ditherMethod = DitherMethod.FloydSteinberg; + img.resize(mg); + img.quantize(qs); + img.write(MagickFormat.Pnm, (data) => { + ws.send(zlib(new Memory(data)).bytes); + }); + }); + await new Promise((resolve) => { ws.addEventListener("message", (e) => { const msg = JSON.parse(e.data); diff --git a/contrib/remote-display-webext/manifest.json b/contrib/remote-display-webext/manifest.json index 994a37df..a1c5bcde 100644 --- a/contrib/remote-display-webext/manifest.json +++ b/contrib/remote-display-webext/manifest.json @@ -4,11 +4,13 @@ "version": "0.1", "description": "Plato Remote Display", "background": { - "scripts": ["background.js"] + "scripts": ["background.js"], + "type": "module" }, "browser_action": { "default_popup": "popup.html" }, + "content_security_policy": "script-src 'self' 'wasm-unsafe-eval' https://esm.sh; object-src 'self'", "permissions": [ "tabs", "", From 7f7aedf216f9c98665304ead2132e7f34ea645c0 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Thu, 14 Mar 2024 20:29:19 -0400 Subject: [PATCH 24/79] Remote display server container --- contrib/Containerfile.remotedisplayserver | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 contrib/Containerfile.remotedisplayserver diff --git a/contrib/Containerfile.remotedisplayserver b/contrib/Containerfile.remotedisplayserver new file mode 100644 index 00000000..b3a8a3b2 --- /dev/null +++ b/contrib/Containerfile.remotedisplayserver @@ -0,0 +1,6 @@ +FROM denoland/deno:alpine-1.41.2 +WORKDIR /app +ADD ./contrib/remote-display-server.ts . +RUN deno cache remote-display-server.ts +EXPOSE 8222 +CMD ["run", "-A", "remote-display-server.ts"] \ No newline at end of file From c5ef232b965084d71ca357b88a4dcc41802714dd Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Sun, 17 Mar 2024 15:28:01 -0400 Subject: [PATCH 25/79] Remote display over MQTT --- contrib/Containerfile.remotedisplayserver | 6 - contrib/remote-display-server.ts | 114 ----------- contrib/remote-display-webext/background.js | 114 +++++------ contrib/remote-display-webext/popup.html | 33 +++- contrib/remote-display-webext/popup.js | 13 +- crates/core/Cargo.toml | 5 +- crates/core/src/settings/mod.rs | 20 +- crates/core/src/view/mod.rs | 3 +- crates/core/src/view/remote_display/mod.rs | 201 ++++++++++---------- 9 files changed, 210 insertions(+), 299 deletions(-) delete mode 100644 contrib/Containerfile.remotedisplayserver delete mode 100644 contrib/remote-display-server.ts diff --git a/contrib/Containerfile.remotedisplayserver b/contrib/Containerfile.remotedisplayserver deleted file mode 100644 index b3a8a3b2..00000000 --- a/contrib/Containerfile.remotedisplayserver +++ /dev/null @@ -1,6 +0,0 @@ -FROM denoland/deno:alpine-1.41.2 -WORKDIR /app -ADD ./contrib/remote-display-server.ts . -RUN deno cache remote-display-server.ts -EXPOSE 8222 -CMD ["run", "-A", "remote-display-server.ts"] \ No newline at end of file diff --git a/contrib/remote-display-server.ts b/contrib/remote-display-server.ts deleted file mode 100644 index dd1f1a04..00000000 --- a/contrib/remote-display-server.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { Application, Router } from "https://deno.land/x/oak@v13.0.0/mod.ts"; - -const app = new Application(); -const router = new Router(); - -let deviceSocket: WebSocket | undefined; -let browserSocket: WebSocket | undefined; -let deviceWidth = 0; -let deviceHeight = 0; - -// #region browser socket -router.get("/browser", (ctx) => { - if (browserSocket) { - browserSocket.close(); - console.log("Closing previous browser connection"); - } - browserSocket = ctx.upgrade(); - console.log("Browser starting connection"); - - browserSocket.onopen = () => { - console.log("Browser connected"); - if (!deviceSocket || !deviceWidth || !deviceHeight) return; - browserSocket?.send( - JSON.stringify({ - type: "size", - value: { width: deviceWidth, height: deviceHeight }, - }), - ); - }; - - browserSocket.onclose = () => { - browserSocket = undefined; - console.log("Browser disconnected"); - }; - - browserSocket.onmessage = (m) => { - if (m.data instanceof ArrayBuffer || m.data instanceof Blob) { - console.log("Browser sends image"); - } else { - console.log(`Browser sends ${m.data}`); - } - try { - deviceSocket?.send(m.data); - } catch (e) { - console.error("Error sending to device: ", e); - deviceSocket = undefined; - } - }; -}); -// #endregion - -// #region device socket -router.get("/device", (ctx) => { - if (deviceSocket) { - ctx.response.status = 400; - ctx.response.body = "Only one device connection allowed"; - return; - } - deviceSocket = ctx.upgrade(); - console.log("Device starting connection"); - - deviceSocket.onopen = () => { - console.log("Device connected"); - }; - - deviceSocket.onclose = () => { - deviceSocket = undefined; - console.log("Device disconnected"); - }; - - deviceSocket.onmessage = (m) => { - if (m.data instanceof ArrayBuffer) return; - console.log(`Device sends ${m.data}`); - const msg = JSON.parse(m.data); - switch (msg.type) { - case "size": - deviceWidth = msg.value.width; - deviceHeight = msg.value.height; - /* falls through */ - default: - try { - browserSocket?.send(m.data); - } catch (e) { - console.error("Error sending to browser: ", e); - browserSocket = undefined; - } - } - }; -}); -// #endregion - -const password = Deno.env.get("REMOTE_DISPLAY_PASSWORD"); -if (password) { - app.use(async (ctx, next) => { - const auth = ctx.request.headers.get("authorization"); - const expectedAuth = `Basic ${btoa(`plato:${password}`)}`; - if (auth !== expectedAuth) { - console.error(`Unauthorized access from ${ctx.request.ip}`); - ctx.response.status = 401; - ctx.response.headers.set("WWW-Authenticate", "Basic"); - ctx.response.body = "Unauthorized"; - return; - } - await next(); - }); - console.log("Password protected with username 'plato'"); -} - -app.use(router.routes()); -app.use(router.allowedMethods()); - -const port = parseInt(Deno.env.get("PORT") || "8222"); -console.log("Listening at port " + port); -await app.listen({ port }); diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 8e6aab7c..a4c54068 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -167,7 +167,7 @@ async function offsetContrastFilter(offset) { let scaleFactor = 1; let deviceWidth = 0; let deviceHeight = 0; -let ws; +let mq; import { ColorSpace, @@ -179,6 +179,9 @@ import { MagickGeometry } from "https://esm.sh/@imagemagick/magick-wasm@0.0.28"; import { Foras, Memory, zlib } from "https://esm.sh/@hazae41/foras@2.1.4"; +import mqtt from "https://esm.sh/mqtt@5.4.0"; +const connectMq = mqtt.connect; + const wasmFile = "https://esm.sh/@imagemagick/magick-wasm@0.0.28/dist/magick.wasm"; await fetch(wasmFile, { cache: "force-cache" }) @@ -186,8 +189,24 @@ await fetch(wasmFile, { cache: "force-cache" }) .then(a => initializeImageMagick(a)); await Foras.initBundledOnce(); +let topic = "remote-display-webext"; +function send(msg) { + if (mq?.connected) { + mq.publish(`${topic}/device`, JSON.stringify(msg)); + } +} + +function sendNotice(notice) { + send({ type: "notify", value: notice }); +} + async function sendImage() { - if (!ws || ws.readyState !== WebSocket.OPEN) return; + if (!mq?.connected) return; + if (!deviceWidth || !deviceHeight) { + send({ type: "updateSize" }); + console.log("cannot update image without size"); + return; + } console.log("capturing"); const { id } = await currentTab(); const dataUrl = await browser.tabs.captureTab(id, { @@ -207,25 +226,22 @@ async function sendImage() { img.resize(mg); img.quantize(qs); img.write(MagickFormat.Pnm, (data) => { - ws.send(zlib(new Memory(data)).bytes); + mq.publish(`${topic}/device`, zlib(new Memory(data)).bytes); }); }); await new Promise((resolve) => { - ws.addEventListener("message", (e) => { - const msg = JSON.parse(e.data); + const updated = (_, m) => { + const msg = JSON.parse(m.toString()); + mq.off("message", updated); if (msg.type === "displayUpdated") { resolve(); } - }, { once: true }); + } + mq.on("message", updated); }); } -function sendNotice(notice) { - if (!ws || ws.readyState !== WebSocket.OPEN) return; - ws.send(JSON.stringify({ type: "notify", value: notice })); -} - let timeout; browser.tabs.onUpdated.addListener(async (_id, changeInfo, tab) => { if (!tab.active) return; @@ -243,9 +259,8 @@ browser.tabs.onUpdated.addListener(async (_id, changeInfo, tab) => { }, 1000); }); -async function onMessage(e) { - console.log("message", e); - const msg = JSON.parse(e.data); +async function onMessage(msg) { + console.log("message", msg); switch (msg.type) { case "size": { const { width, height } = msg.value; @@ -344,7 +359,7 @@ async function onMessage(e) { case "holdFingerShort": await resizeViewport(deviceWidth / scaleFactor, deviceHeight / scaleFactor); await sendImage(); - await ws.send(JSON.stringify({ type: "refreshDisplay" })); + send({ type: "refreshDisplay" }); break; case "holdFingerLong": { const [{ x, y }] = msg.value; @@ -388,59 +403,50 @@ async function onMessage(e) { } const defaultConfig = { - wsUrl: "ws://localhost:8222/browser", + wsUrl: "wss://broker.hivemq.com:8884/mqtt", + topic: "remote-display-webext", enabled: false, }; async function getConfig() { - const result = await browser.storage.local.get(["wsUrl", "enabled"]); + const result = await browser.storage.local.get(["wsUrl", "topic", "enabled"]); + topic = result.topic || defaultConfig.topic; await browser.storage.local.set({ ...defaultConfig, ...result }); return { ...defaultConfig, ...result }; } -const RETRY_TIMEOUT = 5000; -let { enabled, wsUrl } = defaultConfig; -function connect(config) { - enabled = config.enabled; - wsUrl = config.wsUrl; - if (!enabled) return; - ws = new WebSocket(wsUrl); - ws.addEventListener("open", () => { - console.log("connected"); - }); - ws.addEventListener("close", () => { - if (!enabled) return; - console.log("disconnected"); - setTimeout(() => { - getConfig().then(connect).catch(console.error); - }, RETRY_TIMEOUT); - }); - ws.addEventListener("message", onMessage); + +function refreshConnection(config) { + if (config.enabled && !mq?.connected) { + mq = connectMq(config.wsUrl); + mq.subscribe(`${config.topic}/browser`); + mq.on("message", (_topic, message) => onMessage(JSON.parse(message.toString()))); + mq.on("connect", () => { + console.log("connected"); + }); + mq.on("disconnect", () => { + console.log("disconnected"); + }); + } else if (!config.enabled && mq?.connected) { + mq.end(); + } } -getConfig().then(connect).catch(console.error); +getConfig().then(refreshConnection).catch(console.error); browser.storage.local.onChanged.addListener((changes) => { if ( - "enabled" in changes && - changes.enabled.newValue && - !changes.enabled.oldValue && - (!ws || ws.readyState === WebSocket.CLOSED) - ) { - getConfig().then(connect).catch(console.error); - } else if ( - "enabled" in changes && - !changes.enabled.newValue && - changes.enabled.oldValue && - ws + ( + ("wsUrl" in changes && changes.wsUrl.newValue !== changes.wsUrl.oldValue) + || ("topic" in changes && changes.topic.newValue !== changes.topic.oldValue) + ) && + mq?.connected ) { - ws.close(); - } else if ( - "wsUrl" in changes && - changes.wsUrl.newValue !== changes.wsUrl.oldValue && - ws - ) { - ws.close(); + mq.end(); + getConfig().then(refreshConnection).catch(console.error); + } + if ("enabled" in changes && changes.enabled.newValue !== changes.enabled.oldValue) { + getConfig().then(refreshConnection).catch(console.error); } }); diff --git a/contrib/remote-display-webext/popup.html b/contrib/remote-display-webext/popup.html index c0df1d9a..417cfbe5 100644 --- a/contrib/remote-display-webext/popup.html +++ b/contrib/remote-display-webext/popup.html @@ -1,21 +1,34 @@ + Settings + -
- - - - - - - -
+
+ + + + - + + + +
+ + + \ No newline at end of file diff --git a/contrib/remote-display-webext/popup.js b/contrib/remote-display-webext/popup.js index 8282d0f0..41a4e060 100644 --- a/contrib/remote-display-webext/popup.js +++ b/contrib/remote-display-webext/popup.js @@ -1,22 +1,25 @@ const form = document.getElementById('form'); const urlInput = document.getElementById('url'); +const topicInput = document.getElementById('topic'); const enabledInput = document.getElementById('enabled'); form.addEventListener('submit', async (e) => { e.preventDefault(); - + await browser.storage.local.set({ wsUrl: urlInput.value, + topic: topicInput.value, enabled: enabledInput.checked }); - - // deno-lint-ignore no-window-prefix + + // deno-lint-ignore no-window-prefix no-window window.close(); }); (async () => { - const {wsUrl, enabled} = await browser.storage.local.get(['wsUrl', 'enabled']); - + const { wsUrl, topic, enabled } = await browser.storage.local.get(['wsUrl', 'topic', 'enabled']); + urlInput.value = wsUrl || ''; + topicInput.value = topic || ''; enabledInput.checked = enabled || false; })(); diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index b27fd3b2..029bb895 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -42,6 +42,7 @@ chrono = { version = "0.4.34", features = ["serde", "clock"], default-features = image = { version = "0.24.7", features = ["pnm"], default-features = false } url = { version = "2.1.0" } base64 = "0.21.4" -tokio-tungstenite = { version = "0.20.0", features = ["rustls-tls-webpki-roots"] } tokio = { version = "1.32.0", features = ["macros", "rt", "net", "sync"] } -futures-util = { version = "0.3.28", default-features = false, features = ["sink", "std"] } +rumqttc = { version = "0.24.0", features = ["use-rustls", "websocket", "url"] } +rand = "=0.8.5" +webpki-roots = "0.26.1" diff --git a/crates/core/src/settings/mod.rs b/crates/core/src/settings/mod.rs index 69f5e789..f8779901 100644 --- a/crates/core/src/settings/mod.rs +++ b/crates/core/src/settings/mod.rs @@ -6,6 +6,8 @@ use std::fmt::{self, Debug}; use std::path::PathBuf; use std::collections::{BTreeMap, HashMap}; use fxhash::FxHashSet; +use rand::distributions::Alphanumeric; +use rand::Rng; use serde::{Serialize, Deserialize}; use crate::metadata::{SortMethod, TextAlign}; use crate::frontlight::LightLevels; @@ -197,14 +199,24 @@ impl Default for DictionarySettings { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, rename_all = "kebab-case")] pub struct RemoteDisplaySettings { - pub address: String + pub address: String, + pub topic: String, } impl Default for RemoteDisplaySettings { fn default() -> Self { - RemoteDisplaySettings { - address: "ws://localhost:8222/device".to_string() - } + let topic: String = rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(16) + .map(char::from) + .collect(); + let client_id: String = rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(16) + .map(char::from) + .collect(); + let address = format!("mqtts://broker.hivemq.com?client_id={}", client_id); + RemoteDisplaySettings { address, topic } } } diff --git a/crates/core/src/view/mod.rs b/crates/core/src/view/mod.rs index 3c9805f4..fae33059 100644 --- a/crates/core/src/view/mod.rs +++ b/crates/core/src/view/mod.rs @@ -364,7 +364,8 @@ pub enum Event { Back, Quit, WakeUp, - UpdateRemoteView(Box>) + UpdateRemoteView(Box>), + SendRemoteViewSize } #[derive(Debug, Clone, Eq, PartialEq)] diff --git a/crates/core/src/view/remote_display/mod.rs b/crates/core/src/view/remote_display/mod.rs index 9b9375a4..28531624 100644 --- a/crates/core/src/view/remote_display/mod.rs +++ b/crates/core/src/view/remote_display/mod.rs @@ -7,20 +7,21 @@ use crate::input::{ButtonCode, ButtonStatus, DeviceEvent}; use crate::view::{Bus, Event, Hub, RenderData, RenderQueue, View}; use crate::view::{Id, ID_FEEDER}; use anyhow::Error; -use base64::{engine::general_purpose, Engine as _}; use flate2::bufread::ZlibDecoder; -use futures_util::{SinkExt, StreamExt}; use image::codecs::pnm::PnmDecoder; use image::ImageDecoder; +use rumqttc::tokio_rustls::rustls::ClientConfig; +use rumqttc::{ + AsyncClient, ConnAck, ConnectReturnCode, Incoming, MqttOptions, QoS, TlsConfiguration, + Transport, +}; use serde::Deserialize; use serde_json::{json, Value}; use std::io::Read; -use std::sync::mpsc as std_mpsc; +use std::sync::{mpsc as std_mpsc, Arc}; use std::thread::spawn; +use std::time::Duration; use tokio::sync::mpsc as tokio_mpsc; -use tokio_tungstenite::tungstenite::client::IntoClientRequest; -use tokio_tungstenite::{connect_async, tungstenite::protocol::Message}; -use url::Url; #[derive(Debug, Clone)] enum SocketEvent { @@ -33,83 +34,90 @@ enum SocketEvent { enum ServerMessage { Notify(String), RefreshDisplay, + UpdateSize, } #[tokio::main(flavor = "current_thread")] async fn display_connection( event_tx: std_mpsc::Sender, mut socket_rx: tokio_mpsc::Receiver, - url: Url, + address: String, + topic: String, ) -> Result<(), Box> { - let username = url.username(); - let password = url.password().unwrap_or(""); - - let mut request = url.clone().into_client_request()?; - match username { - "" => {} - username => { - request.headers_mut().append( - "Authorization", - format!( - "Basic {}", - general_purpose::STANDARD - .encode(format!("{}:{}", username, password).as_bytes()) - ) - .parse()?, - ); + let mut mqo = MqttOptions::parse_url(address)?; + mqo.set_keep_alive(Duration::from_secs(30)); + mqo.set_max_packet_size(1_048_576, 1_048_576); + let root_store = rumqttc::tokio_rustls::rustls::RootCertStore { + roots: webpki_roots::TLS_SERVER_ROOTS.iter().cloned().collect(), + }; + let cc = ClientConfig::builder() + .with_root_certificates(root_store) + .with_no_client_auth(); + let tlsc = TlsConfiguration::Rustls(Arc::new(cc)); + match mqo.transport() { + Transport::Tls(..) => { + mqo.set_transport(Transport::Tls(tlsc)); + } + Transport::Wss(..) => { + mqo.set_transport(Transport::Wss(tlsc)); } + _ => {} } - - let (mut socket, _) = connect_async(request).await?; + let (client, mut eventloop) = AsyncClient::new(mqo, 10); + let sub_topic = topic.clone() + "/device"; + let pub_topic = topic.clone() + "/browser"; + client.subscribe(sub_topic, QoS::ExactlyOnce).await?; event_tx.send(Event::Notify("Connected".to_string()))?; loop { tokio::select! { - Some(event) = socket_rx.recv() => { - match event { - SocketEvent::Finished => { break; } - SocketEvent::SendJSON(val) => { - socket.send(Message::Text(val.to_string())).await?; + Some(socket_event) = socket_rx.recv() => { + match socket_event { + SocketEvent::Finished => { + client.disconnect().await?; + break; + } + SocketEvent::SendJSON(value) => { + client.publish(pub_topic.clone(), QoS::ExactlyOnce, false, value.to_string()).await?; } } } - Some(msg) = socket.next() => { - let text = match msg { - Ok(Message::Text(text)) => text, - Ok(Message::Binary(bin)) => { - event_tx.send(Event::UpdateRemoteView(Box::new(bin)))?; - continue; - }, - Ok(Message::Close(_)) => { break; } - Ok(_) => { continue; } - Err(e) => { - event_tx.send(Event::Notify(e.to_string()))?; - break; - } - }; + event = eventloop.poll() => { + match event { + Ok(rumqttc::Event::Incoming(Incoming::Publish(p))) => { + let payload = p.payload; - let server_msg = match serde_json::from_str::(&text) { - Ok(sm) => sm, - Err(e) => { - println!("{}", e); - println!("contents: {}", text); - event_tx.send(Event::Notify("Invalid message from server".to_string()))?; - continue; + if !payload.starts_with(&[123, 34]) { + event_tx.send(Event::UpdateRemoteView(Box::new(payload.to_vec())))?; + continue; + } + let message = serde_json::from_slice::(&payload)?; + event_tx.send(match message { + ServerMessage::Notify(message) => { + Event::Notify(message) + } + ServerMessage::RefreshDisplay => { + Event::Update(UpdateMode::Full) + } + ServerMessage::UpdateSize => { + Event::SendRemoteViewSize + } + })?; } - }; - - match server_msg { - ServerMessage::Notify(msg) => { - event_tx.send(Event::Notify(msg))?; + Ok(rumqttc::Event::Incoming( + Incoming::ConnAck(ConnAck { session_present: _, code: ConnectReturnCode::Success })) + ) => { + event_tx.send(Event::SendRemoteViewSize)?; } - ServerMessage::RefreshDisplay => { - event_tx.send(Event::Update(UpdateMode::Full))?; + Ok(..) => {} + Err(e) => { + println!("{}", e); + event_tx.send(Event::Notify(e.to_string()))?; } } } } } event_tx.send(Event::Notify("Disconnected".to_string()))?; - socket.close(None).await?; Ok(()) } @@ -119,7 +127,7 @@ pub struct RemoteDisplay { rect: Rectangle, children: Vec>, pixmap: Pixmap, - socket_tx: tokio_mpsc::Sender, + message_tx: tokio_mpsc::Sender, } impl RemoteDisplay { @@ -133,44 +141,28 @@ impl RemoteDisplay { let children = Vec::new(); rq.add(RenderData::new(id, rect, UpdateMode::Full)); - let tx = hub.clone(); - let my_tx = hub.clone(); - let address = context.settings.remote_display.address.clone(); - let (socket_tx, socket_rx) = tokio_mpsc::channel(10); - spawn(move || { - let url = match Url::parse(&address) { - Ok(url) => url, - Err(e) => { - my_tx.send(Event::Back).ok(); - tx.send(Event::Notify(e.to_string())).ok(); - return; - } - }; - match display_connection(tx, socket_rx, url) { - Ok(_) => {} + let topic = context.settings.remote_display.topic.clone(); + + let (message_tx, message_rx) = tokio_mpsc::channel(16); + let event_tx = hub.clone(); + + spawn( + move || match display_connection(event_tx.clone(), message_rx, address, topic) { + Ok(..) => {} Err(e) => { - my_tx.send(Event::Back).ok(); - my_tx.send(Event::Notify(e.to_string())).ok(); + event_tx.send(Event::Back).ok(); + event_tx.send(Event::Notify(e.to_string())).ok(); } - } - }); - socket_tx - .try_send(SocketEvent::SendJSON(json!({ - "type": "size", - "value": { - "width": rect.width(), - "height": rect.height(), - } - }))) - .ok(); + }, + ); RemoteDisplay { id, rect, children, - socket_tx, pixmap: Pixmap::new(rect.width(), rect.height()), + message_tx, } } @@ -182,7 +174,7 @@ impl RemoteDisplay { let mut pixmap = Pixmap::new(width, height); dec.read_image(&mut pixmap.data_mut())?; self.pixmap = pixmap; - self.socket_tx + self.message_tx .try_send(SocketEvent::SendJSON(json!({ "type": "displayUpdated", }))) @@ -206,12 +198,12 @@ impl View for RemoteDisplay { start: _start, end: _end, }) => { - self.socket_tx.try_send(SocketEvent::Finished).ok(); + self.message_tx.try_send(SocketEvent::Finished).ok(); bus.push_back(Event::Back); true } Event::Gesture(ge) => { - self.socket_tx + self.message_tx .try_send(SocketEvent::SendJSON(serde_json::to_value(ge).unwrap())) .ok(); true @@ -227,7 +219,7 @@ impl View for RemoteDisplay { ButtonStatus::Released => "released", ButtonStatus::Repeated => "repeated", }; - self.socket_tx + self.message_tx .try_send(SocketEvent::SendJSON(json!({ "type": "button", "value": { @@ -250,6 +242,18 @@ impl View for RemoteDisplay { rq.add(RenderData::new(self.id, self.rect, UpdateMode::Gui)); true } + Event::SendRemoteViewSize => { + self.message_tx + .try_send(SocketEvent::SendJSON(json!({ + "type": "size", + "value": { + "width": self.rect.width(), + "height": self.rect.height(), + } + }))) + .ok(); + true + } Event::Update(UpdateMode::Full) => { rq.add(RenderData::new(self.id, self.rect, UpdateMode::Full)); true @@ -286,17 +290,8 @@ impl View for RemoteDisplay { self.children[i].resize(rect, hub, rq, context); } - self.socket_tx - .try_send(SocketEvent::SendJSON(json!({ - "type": "size", - "value": { - "width": rect.width(), - "height": rect.height(), - } - }))) - .ok(); - self.rect = rect; + hub.send(Event::SendRemoteViewSize).unwrap(); rq.add(RenderData::new(self.id, self.rect, UpdateMode::Full)); } From 4b581dab0b1c93378ef1bf33fc465ace0d399dc7 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Sun, 17 Mar 2024 17:37:16 -0400 Subject: [PATCH 26/79] Subscribe on reconnect and handle failed connections gracefully --- crates/core/src/view/remote_display/mod.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/core/src/view/remote_display/mod.rs b/crates/core/src/view/remote_display/mod.rs index 28531624..a9bd0510 100644 --- a/crates/core/src/view/remote_display/mod.rs +++ b/crates/core/src/view/remote_display/mod.rs @@ -17,6 +17,7 @@ use rumqttc::{ }; use serde::Deserialize; use serde_json::{json, Value}; +use tokio::time::sleep; use std::io::Read; use std::sync::{mpsc as std_mpsc, Arc}; use std::thread::spawn; @@ -66,8 +67,6 @@ async fn display_connection( let (client, mut eventloop) = AsyncClient::new(mqo, 10); let sub_topic = topic.clone() + "/device"; let pub_topic = topic.clone() + "/browser"; - client.subscribe(sub_topic, QoS::ExactlyOnce).await?; - event_tx.send(Event::Notify("Connected".to_string()))?; loop { tokio::select! { Some(socket_event) = socket_rx.recv() => { @@ -77,7 +76,7 @@ async fn display_connection( break; } SocketEvent::SendJSON(value) => { - client.publish(pub_topic.clone(), QoS::ExactlyOnce, false, value.to_string()).await?; + client.publish(pub_topic.clone(), QoS::AtMostOnce, false, value.to_string()).await?; } } } @@ -106,12 +105,15 @@ async fn display_connection( Ok(rumqttc::Event::Incoming( Incoming::ConnAck(ConnAck { session_present: _, code: ConnectReturnCode::Success })) ) => { + event_tx.send(Event::Notify("Connected".to_string()))?; + client.subscribe(sub_topic.clone(), QoS::AtMostOnce).await?; event_tx.send(Event::SendRemoteViewSize)?; } Ok(..) => {} Err(e) => { println!("{}", e); event_tx.send(Event::Notify(e.to_string()))?; + sleep(Duration::from_millis(2000)).await; } } } From d4452403fe5a4f8195b8fb655cae5c0f32c9aed5 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:18:29 -0400 Subject: [PATCH 27/79] Add connection and last will notification for browser --- contrib/remote-display-webext/background.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index a4c54068..84833fae 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -418,10 +418,17 @@ async function getConfig() { function refreshConnection(config) { if (config.enabled && !mq?.connected) { - mq = connectMq(config.wsUrl); + mq = connectMq(config.wsUrl, { + will: { + topic: `${config.topic}/device`, + payload: JSON.stringify({ type: "notify", value: "Browser disconnected" }), + qos: 0, + } + }); mq.subscribe(`${config.topic}/browser`); mq.on("message", (_topic, message) => onMessage(JSON.parse(message.toString()))); mq.on("connect", () => { + sendNotice("Browser connected"); console.log("connected"); }); mq.on("disconnect", () => { From b95a965a7be357e4bca184592e1847e7ef8c9233 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Tue, 26 Mar 2024 20:25:16 -0400 Subject: [PATCH 28/79] Use CBOR for message format --- Cargo.lock | 625 +++++++++++++++----- contrib/remote-display-webext/background.js | 11 +- crates/core/Cargo.toml | 2 + crates/core/src/view/remote_display/mod.rs | 67 ++- 4 files changed, 518 insertions(+), 187 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd204db9..8df24817 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,9 +54,38 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" + +[[package]] +name = "async-tungstenite" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef0f8d64ef9351752fbe5462f242c625d9c4910d2bc3f7ec44c43857ca123f5d" +dependencies = [ + "futures-io", + "futures-util", + "log", + "pin-project-lite", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.25.0", + "tungstenite", +] + +[[package]] +name = "async_io_stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" +dependencies = [ + "futures", + "pharos", + "rustc_version", + "tokio", +] [[package]] name = "atlatl" @@ -144,15 +173,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.15.3" +version = "3.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" [[package]] name = "bytemuck" -version = "1.14.3" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ef034f05691a48569bd920a96c81b9d91bbad1ab5ac7c4616c1f6ef36cb79f" +checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" [[package]] name = "byteorder" @@ -189,10 +218,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.87" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3286b845d0fccbdd15af433f61c5970e711987036cb468f437ff6badd70f4e24" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" dependencies = [ + "jobserver", "libc", ] @@ -210,15 +240,42 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "chrono" -version = "0.4.34" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" +checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", "serde", - "windows-targets 0.52.3", + "windows-targets 0.52.4", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", ] [[package]] @@ -259,6 +316,16 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "coset" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c7f688ee58418b8614848a05bf392cc622f92451e79998c131b9d22c91abd9" +dependencies = [ + "ciborium", + "ciborium-io", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -283,6 +350,12 @@ version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -295,9 +368,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "deranged" @@ -382,6 +455,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -397,6 +481,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -404,6 +503,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -412,12 +512,34 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.30" @@ -436,8 +558,10 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -506,16 +630,16 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" +checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" dependencies = [ "bytes", "fnv", "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", "indexmap", "slab", "tokio", @@ -524,16 +648,20 @@ dependencies = [ ] [[package]] -name = "hashbrown" -version = "0.14.3" +name = "half" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" +dependencies = [ + "cfg-if", + "crunchy", +] [[package]] -name = "hermit-abi" -version = "0.3.8" +name = "hashbrown" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379dada1584ad501b383485dd706b8afb7a70fcbc7f4da7d780638a5a6124a60" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "hmac" @@ -546,9 +674,20 @@ dependencies = [ [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -562,7 +701,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", "pin-project-lite", ] @@ -589,7 +728,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.12", "http-body", "httparse", "httpdate", @@ -609,11 +748,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", + "http 0.2.12", "hyper", - "rustls", + "rustls 0.21.10", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", ] [[package]] @@ -651,14 +790,13 @@ dependencies = [ [[package]] name = "image" -version = "0.24.7" +version = "0.24.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f3dfdbdd72063086ff443e297b61695500514b1e41095b6fb9a5ab48a70a711" +checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" dependencies = [ "bytemuck", "byteorder", "color_quant", - "num-rational", "num-traits", ] @@ -672,9 +810,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.3" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", "hashbrown", @@ -702,6 +840,15 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "jobserver" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +dependencies = [ + "libc", +] + [[package]] name = "joinery" version = "2.1.0" @@ -710,9 +857,9 @@ checksum = "72167d68f5fce3b8655487b8038691a3c9984ee769590f93f2a631f4ad64e4f5" [[package]] name = "js-sys" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -757,11 +904,21 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "memchr" @@ -787,9 +944,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi", @@ -814,26 +971,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.18" @@ -843,16 +980,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "object" version = "0.32.2" @@ -868,6 +995,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "paragraph-breaker" version = "0.4.4" @@ -903,6 +1036,16 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pharos" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" +dependencies = [ + "futures", + "rustc_version", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -936,11 +1079,12 @@ dependencies = [ "base64", "bitflags 2.4.2", "byteorder", + "bytes", "chrono", + "coset", "downcast-rs", "entities", "flate2", - "futures-util", "fxhash", "globset", "image", @@ -953,20 +1097,22 @@ dependencies = [ "paragraph-breaker", "percent-encoding", "png", + "rand", "rand_core", "rand_xoshiro", "regex", + "rumqttc", "septem", "serde", "serde_json", "thiserror", "titlecase", "tokio", - "tokio-tungstenite", "toml", "unicode-normalization", "url", "walkdir", + "webpki-roots 0.26.1", "xi-unicode", "zip", ] @@ -998,9 +1144,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -1067,9 +1213,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", @@ -1084,9 +1230,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.24" +version = "0.11.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" dependencies = [ "base64", "bytes", @@ -1094,7 +1240,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.12", "http-body", "hyper", "hyper-rustls", @@ -1105,21 +1251,21 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", - "rustls-pemfile", + "rustls 0.21.10", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "system-configuration", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "webpki-roots 0.25.4", "winreg", ] @@ -1138,12 +1284,43 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rumqttc" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1568e15fab2d546f940ed3a21f48bbbd1c494c90c99c4481339364a497f94a9" +dependencies = [ + "async-tungstenite", + "bytes", + "flume", + "futures-util", + "http 1.1.0", + "log", + "rustls-native-certs", + "rustls-pemfile 2.1.1", + "rustls-webpki 0.102.2", + "thiserror", + "tokio", + "tokio-rustls 0.25.0", + "url", + "ws_stream_tungstenite", +] + [[package]] name = "rustc-demangle" version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustls" version = "0.21.10" @@ -1152,10 +1329,37 @@ checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", "ring", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.2", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.1.1", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -1165,6 +1369,22 @@ dependencies = [ "base64", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab" +dependencies = [ + "base64", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ede67b28608b4c60685c7d54122d4400d90f62b40caee7700e700380a390fa8" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -1175,6 +1395,17 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.102.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.17" @@ -1190,6 +1421,21 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "sct" version = "0.7.1" @@ -1223,6 +1469,35 @@ dependencies = [ "version-compare", ] +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" + [[package]] name = "septem" version = "1.1.0" @@ -1352,6 +1627,9 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] [[package]] name = "subtle" @@ -1361,9 +1639,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.50" +version = "2.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f1bdc9872430ce9b75da68329d1c1746faf50ffac5f19e02b71e37ff881ffb" +checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" dependencies = [ "proc-macro2", "quote", @@ -1399,18 +1677,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", @@ -1472,7 +1750,6 @@ dependencies = [ "bytes", "libc", "mio", - "num_cpus", "pin-project-lite", "socket2", "tokio-macros", @@ -1496,23 +1773,19 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.10", "tokio", ] [[package]] -name = "tokio-tungstenite" -version = "0.20.1" +name = "tokio-rustls" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ - "futures-util", - "log", - "rustls", + "rustls 0.22.2", + "rustls-pki-types", "tokio", - "tokio-rustls", - "tungstenite", - "webpki-roots", ] [[package]] @@ -1531,9 +1804,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a9aad4a3066010876e8dcf5a8a06e70a558751117a145c6ce2b82c2e2054290" +checksum = "af06656561d28735e9c1cd63dfd57132c8155426aa6af24f36a00a351f88c48e" dependencies = [ "serde", "serde_spanned", @@ -1552,9 +1825,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.6" +version = "0.22.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1b5fd4128cc8d3e0cb74d4ed9a9cc7c7284becd4df68f5f940e1ad123606f6" +checksum = "18769cd1cec395d70860ceb4d932812a0b4d06b1a4bb336745a4d21b9496e992" dependencies = [ "indexmap", "serde", @@ -1576,9 +1849,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.32" @@ -1596,18 +1881,19 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" +checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" dependencies = [ "byteorder", "bytes", "data-encoding", - "http", + "http 1.1.0", "httparse", "log", "rand", - "rustls", + "rustls 0.22.2", + "rustls-pki-types", "sha1", "thiserror", "url", @@ -1684,9 +1970,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -1709,9 +1995,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1719,9 +2005,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", @@ -1734,9 +2020,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -1746,9 +2032,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1756,9 +2042,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", @@ -1769,15 +2055,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -1789,6 +2075,15 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "webpki-roots" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1826,7 +2121,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.3", + "windows-targets 0.52.4", ] [[package]] @@ -1844,7 +2139,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.3", + "windows-targets 0.52.4", ] [[package]] @@ -1864,17 +2159,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" dependencies = [ - "windows_aarch64_gnullvm 0.52.3", - "windows_aarch64_msvc 0.52.3", - "windows_i686_gnu 0.52.3", - "windows_i686_msvc 0.52.3", - "windows_x86_64_gnu 0.52.3", - "windows_x86_64_gnullvm 0.52.3", - "windows_x86_64_msvc 0.52.3", + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -1885,9 +2180,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" @@ -1897,9 +2192,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" @@ -1909,9 +2204,9 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" @@ -1921,9 +2216,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" @@ -1933,9 +2228,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" [[package]] name = "windows_x86_64_gnullvm" @@ -1945,9 +2240,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" @@ -1957,15 +2252,15 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "winnow" -version = "0.6.2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a4191c47f15cc3ec71fcb4913cb83d58def65dd3787610213c649283b5ce178" +checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" dependencies = [ "memchr", ] @@ -1980,12 +2275,38 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ws_stream_tungstenite" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a198f414f083fb19fcc1bffcb0fa0cf46d33ccfa229adf248cac12c180e91609" +dependencies = [ + "async-tungstenite", + "async_io_stream", + "bitflags 2.4.2", + "futures-core", + "futures-io", + "futures-sink", + "futures-util", + "pharos", + "rustc_version", + "tokio", + "tracing", + "tungstenite", +] + [[package]] name = "xi-unicode" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a67300977d3dc3f8034dae89778f502b6ba20b269527b3223ba59c0cf393bb8a" +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + [[package]] name = "zip" version = "0.6.6" diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 84833fae..f18f3879 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -180,6 +180,7 @@ import { } from "https://esm.sh/@imagemagick/magick-wasm@0.0.28"; import { Foras, Memory, zlib } from "https://esm.sh/@hazae41/foras@2.1.4"; import mqtt from "https://esm.sh/mqtt@5.4.0"; +import { encode, decode } from "https://esm.sh/cborg@4.1.4"; const connectMq = mqtt.connect; const wasmFile = @@ -192,7 +193,7 @@ await Foras.initBundledOnce(); let topic = "remote-display-webext"; function send(msg) { if (mq?.connected) { - mq.publish(`${topic}/device`, JSON.stringify(msg)); + mq.publish(`${topic}/device`, encode(msg)); } } @@ -226,13 +227,13 @@ async function sendImage() { img.resize(mg); img.quantize(qs); img.write(MagickFormat.Pnm, (data) => { - mq.publish(`${topic}/device`, zlib(new Memory(data)).bytes); + send({ type: "updateDisplay", value: zlib(new Memory(data)).bytes }); }); }); await new Promise((resolve) => { const updated = (_, m) => { - const msg = JSON.parse(m.toString()); + const msg = decode(m); mq.off("message", updated); if (msg.type === "displayUpdated") { resolve(); @@ -421,12 +422,12 @@ function refreshConnection(config) { mq = connectMq(config.wsUrl, { will: { topic: `${config.topic}/device`, - payload: JSON.stringify({ type: "notify", value: "Browser disconnected" }), + payload: encode({ type: "notify", value: "Browser disconnected" }), qos: 0, } }); mq.subscribe(`${config.topic}/browser`); - mq.on("message", (_topic, message) => onMessage(JSON.parse(message.toString()))); + mq.on("message", (_topic, message) => onMessage(decode(message))); mq.on("connect", () => { sendNotice("Browser connected"); console.log("connected"); diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 029bb895..5eaf6ce4 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -46,3 +46,5 @@ tokio = { version = "1.32.0", features = ["macros", "rt", "net", "sync"] } rumqttc = { version = "0.24.0", features = ["use-rustls", "websocket", "url"] } rand = "=0.8.5" webpki-roots = "0.26.1" +coset = "0.3.6" +bytes = "^1.5" diff --git a/crates/core/src/view/remote_display/mod.rs b/crates/core/src/view/remote_display/mod.rs index a9bd0510..a0aefc5f 100644 --- a/crates/core/src/view/remote_display/mod.rs +++ b/crates/core/src/view/remote_display/mod.rs @@ -7,6 +7,8 @@ use crate::input::{ButtonCode, ButtonStatus, DeviceEvent}; use crate::view::{Bus, Event, Hub, RenderData, RenderQueue, View}; use crate::view::{Id, ID_FEEDER}; use anyhow::Error; +use bytes::Buf; +use coset::cbor::{cbor, de::from_reader, ser::into_writer, Value}; use flate2::bufread::ZlibDecoder; use image::codecs::pnm::PnmDecoder; use image::ImageDecoder; @@ -16,18 +18,17 @@ use rumqttc::{ Transport, }; use serde::Deserialize; -use serde_json::{json, Value}; -use tokio::time::sleep; use std::io::Read; use std::sync::{mpsc as std_mpsc, Arc}; use std::thread::spawn; use std::time::Duration; use tokio::sync::mpsc as tokio_mpsc; +use tokio::time::sleep; #[derive(Debug, Clone)] enum SocketEvent { Finished, - SendJSON(Value), + SendMessage(Value), } #[derive(Debug, Clone, Deserialize)] @@ -36,6 +37,7 @@ enum ServerMessage { Notify(String), RefreshDisplay, UpdateSize, + UpdateDisplay(Vec), } #[tokio::main(flavor = "current_thread")] @@ -75,21 +77,17 @@ async fn display_connection( client.disconnect().await?; break; } - SocketEvent::SendJSON(value) => { - client.publish(pub_topic.clone(), QoS::AtMostOnce, false, value.to_string()).await?; + SocketEvent::SendMessage(value) => { + let mut writer = Vec::new(); + into_writer(&value, &mut writer)?; + client.publish(pub_topic.clone(), QoS::AtMostOnce, false, writer).await?; } } } event = eventloop.poll() => { match event { Ok(rumqttc::Event::Incoming(Incoming::Publish(p))) => { - let payload = p.payload; - - if !payload.starts_with(&[123, 34]) { - event_tx.send(Event::UpdateRemoteView(Box::new(payload.to_vec())))?; - continue; - } - let message = serde_json::from_slice::(&payload)?; + let message = from_reader(p.payload.chunk())?; event_tx.send(match message { ServerMessage::Notify(message) => { Event::Notify(message) @@ -100,6 +98,9 @@ async fn display_connection( ServerMessage::UpdateSize => { Event::SendRemoteViewSize } + ServerMessage::UpdateDisplay(data) => { + Event::UpdateRemoteView(Box::new(data)) + } })?; } Ok(rumqttc::Event::Incoming( @@ -177,9 +178,9 @@ impl RemoteDisplay { dec.read_image(&mut pixmap.data_mut())?; self.pixmap = pixmap; self.message_tx - .try_send(SocketEvent::SendJSON(json!({ - "type": "displayUpdated", - }))) + .try_send(SocketEvent::SendMessage(cbor!({ + "type" => "displayUpdated", + })?)) .ok(); Ok(()) } @@ -206,7 +207,7 @@ impl View for RemoteDisplay { } Event::Gesture(ge) => { self.message_tx - .try_send(SocketEvent::SendJSON(serde_json::to_value(ge).unwrap())) + .try_send(SocketEvent::SendMessage(cbor!(ge).unwrap())) .ok(); true } @@ -222,13 +223,16 @@ impl View for RemoteDisplay { ButtonStatus::Repeated => "repeated", }; self.message_tx - .try_send(SocketEvent::SendJSON(json!({ - "type": "button", - "value": { - "button": button, - "status": status, - } - }))) + .try_send(SocketEvent::SendMessage( + cbor!({ + "type" => "button", + "value" => { + "button" => button, + "status" => status, + } + }) + .unwrap(), + )) .ok(); true } @@ -246,13 +250,16 @@ impl View for RemoteDisplay { } Event::SendRemoteViewSize => { self.message_tx - .try_send(SocketEvent::SendJSON(json!({ - "type": "size", - "value": { - "width": self.rect.width(), - "height": self.rect.height(), - } - }))) + .try_send(SocketEvent::SendMessage( + cbor!({ + "type" => "size", + "value" => { + "width" => self.rect.width(), + "height" => self.rect.height(), + } + }) + .unwrap(), + )) .ok(); true } From fbb8fedafe9826b77078c0a135f14de4d473e923 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:56:21 -0400 Subject: [PATCH 29/79] Update size on new browser connection and log display update --- contrib/remote-display-webext/background.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index f18f3879..1856aeb0 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -236,6 +236,7 @@ async function sendImage() { const msg = decode(m); mq.off("message", updated); if (msg.type === "displayUpdated") { + console.log("resolved display update roundtrip") resolve(); } } @@ -430,6 +431,7 @@ function refreshConnection(config) { mq.on("message", (_topic, message) => onMessage(decode(message))); mq.on("connect", () => { sendNotice("Browser connected"); + send({ type: "updateSize" }); console.log("connected"); }); mq.on("disconnect", () => { From 8ccce05f51ae8b8edabc8105de151022f1b70794 Mon Sep 17 00:00:00 2001 From: ewired <37567272+ewired@users.noreply.github.com> Date: Wed, 27 Mar 2024 19:16:21 -0400 Subject: [PATCH 30/79] Use chacha20poly1305 COSE encryption --- Cargo.lock | 71 +++++++++++++++++++++ contrib/remote-display-webext/background.js | 47 +++++++++++--- contrib/remote-display-webext/popup.html | 4 ++ contrib/remote-display-webext/popup.js | 7 +- crates/core/Cargo.toml | 2 + crates/core/src/settings/mod.rs | 9 ++- crates/core/src/view/remote_display/mod.rs | 51 +++++++++++++-- 7 files changed, 174 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8df24817..4da7984e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + [[package]] name = "aes" version = "0.8.4" @@ -238,6 +248,30 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + [[package]] name = "chrono" version = "0.4.35" @@ -286,6 +320,7 @@ checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", + "zeroize", ] [[package]] @@ -363,6 +398,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", + "rand_core", "typenum", ] @@ -663,6 +699,12 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hmac" version = "0.12.1" @@ -995,6 +1037,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "openssl-probe" version = "0.1.5" @@ -1080,6 +1128,7 @@ dependencies = [ "bitflags 2.4.2", "byteorder", "bytes", + "chacha20poly1305", "chrono", "coset", "downcast-rs", @@ -1087,6 +1136,7 @@ dependencies = [ "flate2", "fxhash", "globset", + "hex", "image", "indexmap", "kl-hyphenate", @@ -1130,6 +1180,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1933,6 +1994,16 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" diff --git a/contrib/remote-display-webext/background.js b/contrib/remote-display-webext/background.js index 1856aeb0..023cf34b 100644 --- a/contrib/remote-display-webext/background.js +++ b/contrib/remote-display-webext/background.js @@ -181,6 +181,11 @@ import { import { Foras, Memory, zlib } from "https://esm.sh/@hazae41/foras@2.1.4"; import mqtt from "https://esm.sh/mqtt@5.4.0"; import { encode, decode } from "https://esm.sh/cborg@4.1.4"; +import { Encrypt0Message } from "https://esm.sh/@ldclabs/cose-ts@1.1.1/encrypt0?dev"; +import { hexToBytes } from "https://esm.sh/@ldclabs/cose-ts@1.1.1/utils?dev"; +import { Header } from "https://esm.sh/@ldclabs/cose-ts@1.1.1/header?dev"; +import { ChaCha20Poly1305Key } from "https://esm.sh/@ldclabs/cose-ts@1.1.1/chacha20poly1305?dev"; +import * as iana from "https://esm.sh/@ldclabs/cose-ts@1.1.1/iana?dev"; const connectMq = mqtt.connect; const wasmFile = @@ -191,10 +196,33 @@ await fetch(wasmFile, { cache: "force-cache" }) await Foras.initBundledOnce(); let topic = "remote-display-webext"; -function send(msg) { +let key; + +async function encodeCipher(msg) { + if (!key) { + throw new Error("no key"); + } + const nonce = new Uint8Array(12); + crypto.getRandomValues(nonce); + return await new Encrypt0Message( + encode(msg), + new Header().setParam(iana.HeaderParameterAlg, iana.AlgorithmChaCha20Poly1305), + new Header().setParam(iana.HeaderParameterIV, nonce), + ).toBytes(key); +} + +async function send(msg) { if (mq?.connected) { - mq.publish(`${topic}/device`, encode(msg)); + mq.publish(`${topic}/device`, await encodeCipher(msg)); + } +} + +async function decodeCipher(msg) { + if (!key) { + throw new Error("no key"); } + const emsg = await Encrypt0Message.fromBytes(key, new Uint8Array(msg)); + return decode(emsg.payload); } function sendNotice(notice) { @@ -232,8 +260,8 @@ async function sendImage() { }); await new Promise((resolve) => { - const updated = (_, m) => { - const msg = decode(m); + const updated = async (_, m) => { + const msg = await decodeCipher(m); mq.off("message", updated); if (msg.type === "displayUpdated") { console.log("resolved display update roundtrip") @@ -408,27 +436,30 @@ const defaultConfig = { wsUrl: "wss://broker.hivemq.com:8884/mqtt", topic: "remote-display-webext", enabled: false, + key: "", }; async function getConfig() { - const result = await browser.storage.local.get(["wsUrl", "topic", "enabled"]); + const result = await browser.storage.local.get(["wsUrl", "topic", "enabled", "key"]); topic = result.topic || defaultConfig.topic; + const hexKey = result.key || defaultConfig.key; + key = ChaCha20Poly1305Key.fromSecret(hexToBytes(hexKey)); await browser.storage.local.set({ ...defaultConfig, ...result }); return { ...defaultConfig, ...result }; } -function refreshConnection(config) { +async function refreshConnection(config) { if (config.enabled && !mq?.connected) { mq = connectMq(config.wsUrl, { will: { topic: `${config.topic}/device`, - payload: encode({ type: "notify", value: "Browser disconnected" }), + payload: await encodeCipher({ type: "notify", value: "Browser disconnected" }), qos: 0, } }); mq.subscribe(`${config.topic}/browser`); - mq.on("message", (_topic, message) => onMessage(decode(message))); + mq.on("message", (_topic, message) => decodeCipher(message).then(onMessage)); mq.on("connect", () => { sendNotice("Browser connected"); send({ type: "updateSize" }); diff --git a/contrib/remote-display-webext/popup.html b/contrib/remote-display-webext/popup.html index 417cfbe5..53b92d42 100644 --- a/contrib/remote-display-webext/popup.html +++ b/contrib/remote-display-webext/popup.html @@ -18,6 +18,10 @@ +