From 0feda95a4090bf7bd8877d42092958d353b7830b Mon Sep 17 00:00:00 2001 From: Francesco Petri Date: Tue, 1 Jul 2025 19:45:19 +0200 Subject: [PATCH 1/3] Add revamped version of TTS output Thanking Arne Hasselbring for the idea of making a separate module that communicates via tokio. What I can do as a novice is still limited, but this should make it easier to apply improvements later on. This version uses a basic TTS library that interface with common screen readers on multiple platforms (e.g. Speech Dispatcher). --- Cargo.lock | 271 ++++++++++++++++-- Cargo.toml | 1 + frontend/src/api.js | 36 +++ game_controller_app/src/handlers.rs | 9 + game_controller_core/Cargo.toml | 1 + game_controller_core/src/action.rs | 8 + .../src/actions/finish_set_play.rs | 4 + .../src/actions/global_game_stuck.rs | 4 + game_controller_core/src/actions/goal.rs | 7 + game_controller_core/src/actions/penalize.rs | 13 + .../src/actions/start_set_play.rs | 20 ++ game_controller_core/src/actions/timeout.rs | 9 + .../src/actions/unpenalize.rs | 8 + game_controller_core/src/lib.rs | 12 +- game_controller_core/src/types.rs | 55 ++++ game_controller_runtime/Cargo.toml | 1 + game_controller_runtime/src/lib.rs | 19 +- game_controller_tts/Cargo.toml | 15 + game_controller_tts/src/lib.rs | 62 ++++ 19 files changed, 527 insertions(+), 28 deletions(-) create mode 100644 game_controller_tts/Cargo.toml create mode 100644 game_controller_tts/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 04e7c86..1ba81e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,6 +212,12 @@ dependencies = [ "serde", ] +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + [[package]] name = "block-buffer" version = "0.10.4" @@ -475,6 +481,20 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +[[package]] +name = "cocoa-foundation" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c6234cbb2e4c785b456c0644748b1ac416dd045799740356f8363dfe00c93f7" +dependencies = [ + "bitflags 1.3.2", + "block", + "core-foundation 0.9.4", + "core-graphics-types 0.1.3", + "libc", + "objc", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -507,6 +527,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -530,12 +560,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa95a34622365fa5bbf40b20b75dba8dfa8c94c734aea8ac9a5ca38af14316f1" dependencies = [ "bitflags 2.9.1", - "core-foundation", - "core-graphics-types", + "core-foundation 0.10.1", + "core-graphics-types 0.2.0", "foreign-types", "libc", ] +[[package]] +name = "core-graphics-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "libc", +] + [[package]] name = "core-graphics-types" version = "0.2.0" @@ -543,7 +584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" dependencies = [ "bitflags 2.9.1", - "core-foundation", + "core-foundation 0.10.1", "libc", ] @@ -796,6 +837,27 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clonable" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a36efbb9bfd58e1723780aa04b61aba95ace6a05d9ffabfdb0b43672552f0805" +dependencies = [ + "dyn-clonable-impl", + "dyn-clone", +] + +[[package]] +name = "dyn-clonable-impl" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8671d54058979a37a26f3511fbf8d198ba1aa35ffb202c42587d918d77213a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "dyn-clone" version = "1.0.19" @@ -1058,6 +1120,7 @@ dependencies = [ "serde", "serde_with 2.3.3", "time", + "tokio", "trait_enum", ] @@ -1106,6 +1169,7 @@ dependencies = [ "game_controller_core", "game_controller_msgs", "game_controller_net", + "game_controller_tts", "network-interface", "serde", "serde_repr", @@ -1116,6 +1180,17 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "game_controller_tts" +version = "5.0.0-rc.2" +dependencies = [ + "anyhow", + "speech-dispatcher", + "tokio", + "tokio-util", + "tts", +] + [[package]] name = "gdk" version = "0.18.2" @@ -1561,7 +1636,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.61.2", ] [[package]] @@ -1977,6 +2052,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + [[package]] name = "markup5ever" version = "0.11.0" @@ -2175,6 +2259,16 @@ dependencies = [ "libc", ] +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", + "objc_exception", +] + [[package]] name = "objc-sys" version = "0.3.5" @@ -2389,6 +2483,15 @@ dependencies = [ "objc2-foundation 0.3.1", ] +[[package]] +name = "objc_exception" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4" +dependencies = [ + "cc", +] + [[package]] name = "object" version = "0.36.7" @@ -2416,6 +2519,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "oxilangtag" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f3f87617a86af77fa3691e6350483e7154c2ead9f1261b75130e21ca0f8acb" +dependencies = [ + "serde", +] + [[package]] name = "pango" version = "0.18.3" @@ -3394,6 +3506,26 @@ dependencies = [ "system-deps", ] +[[package]] +name = "speech-dispatcher" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727d53c474ba5ada07784ad7d203cf896a74854cfee0eb32376b00759eb2972" +dependencies = [ + "lazy_static", + "libc", + "speech-dispatcher-sys", +] + +[[package]] +name = "speech-dispatcher-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c3e8acdf2b1f4bb13f1813b40b52f3edf4cc94d8a55fe713a584f672a10388d" +dependencies = [ + "bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -3504,7 +3636,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e59c1f38e657351a2e822eadf40d6a2ad4627b9c25557bc1180ec1b3295ef82" dependencies = [ "bitflags 2.9.1", - "core-foundation", + "core-foundation 0.10.1", "core-graphics", "crossbeam-channel", "dispatch", @@ -3530,8 +3662,8 @@ dependencies = [ "tao-macros", "unicode-segmentation", "url", - "windows", - "windows-core", + "windows 0.61.3", + "windows-core 0.61.2", "windows-version", "x11-dl", ] @@ -3601,7 +3733,7 @@ dependencies = [ "webkit2gtk", "webview2-com", "window-vibrancy", - "windows", + "windows 0.61.3", ] [[package]] @@ -3686,7 +3818,7 @@ dependencies = [ "tauri-utils", "thiserror 2.0.12", "url", - "windows", + "windows 0.61.3", ] [[package]] @@ -3712,7 +3844,7 @@ dependencies = [ "url", "webkit2gtk", "webview2-com", - "windows", + "windows 0.61.3", "wry", ] @@ -4066,6 +4198,29 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tts" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0727c46b3181e4f84e79f970e6a78d3b4054b72b6072e969ea4f07dfa4983ae2" +dependencies = [ + "cocoa-foundation", + "core-foundation 0.9.4", + "dyn-clonable", + "jni", + "lazy_static", + "libc", + "log", + "ndk-context", + "objc", + "oxilangtag", + "speech-dispatcher", + "thiserror 1.0.69", + "wasm-bindgen", + "web-sys", + "windows 0.58.0", +] + [[package]] name = "typeid" version = "1.0.3" @@ -4409,10 +4564,10 @@ checksum = "b542b5cfbd9618c46c2784e4d41ba218c336ac70d44c55e47b251033e7d85601" dependencies = [ "webview2-com-macros", "webview2-com-sys", - "windows", - "windows-core", - "windows-implement", - "windows-interface", + "windows 0.61.3", + "windows-core 0.61.2", + "windows-implement 0.60.0", + "windows-interface 0.59.1", ] [[package]] @@ -4433,8 +4588,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae2d11c4a686e4409659d7891791254cf9286d3cfe0eef54df1523533d22295" dependencies = [ "thiserror 2.0.12", - "windows", - "windows-core", + "windows 0.61.3", + "windows-core 0.61.2", ] [[package]] @@ -4495,6 +4650,16 @@ dependencies = [ "windows-version", ] +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.61.3" @@ -4502,7 +4667,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", - "windows-core", + "windows-core 0.61.2", "windows-future", "windows-link", "windows-numerics", @@ -4514,7 +4679,20 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ - "windows-core", + "windows-core 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement 0.58.0", + "windows-interface 0.58.0", + "windows-result 0.2.0", + "windows-strings 0.1.0", + "windows-targets 0.52.6", ] [[package]] @@ -4523,11 +4701,11 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ - "windows-implement", - "windows-interface", + "windows-implement 0.60.0", + "windows-interface 0.59.1", "windows-link", - "windows-result", - "windows-strings", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -4536,11 +4714,22 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ - "windows-core", + "windows-core 0.61.2", "windows-link", "windows-threading", ] +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "windows-implement" version = "0.60.0" @@ -4552,6 +4741,17 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "windows-interface" version = "0.59.1" @@ -4575,10 +4775,19 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ - "windows-core", + "windows-core 0.61.2", "windows-link", ] +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.3.4" @@ -4588,6 +4797,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result 0.2.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-strings" version = "0.4.2" @@ -4917,8 +5136,8 @@ dependencies = [ "webkit2gtk", "webkit2gtk-sys", "webview2-com", - "windows", - "windows-core", + "windows 0.61.3", + "windows-core 0.61.2", "windows-version", "x11-dl", ] diff --git a/Cargo.toml b/Cargo.toml index 92d7d7b..20f91c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ game_controller_core = { path = "game_controller_core" } game_controller_msgs = { path = "game_controller_msgs" } game_controller_net = { path = "game_controller_net" } game_controller_runtime = { path = "game_controller_runtime" } +game_controller_tts = { path = "game_controller_tts" } network-interface = { version = "1" } serde = { version = "1.0", features = ["derive"] } serde_with = { version = "2.3", features = ["base64", "time_0_3"] } diff --git a/frontend/src/api.js b/frontend/src/api.js index 0f92e7c..205af1a 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -210,6 +210,42 @@ export const syncWithBackend = async () => { } }; +// press M to toggle mute mode +// hold spacebar for hold mode +let mute = false; +let hold = false; + +document.addEventListener('keydown', function(event) { + if (event.key == 'm') { + mute = !mute; + if (window.__TAURI_INTERNALS__) { + invoke("set_mute", { mute: mute }); + } else { + console.log("Set mute to " + mute); + } + } + else if (event.key == ' ') { + hold = true; + if (window.__TAURI_INTERNALS__) { + invoke("set_hold", { hold: hold }); + } else { + console.log("Set hold to " + hold); + } + } +}); + +document.addEventListener('keyup', function(event) { + if (event.key == ' ') { + hold = false; + if (window.__TAURI_INTERNALS__) { + invoke("set_hold", { hold: hold }); + } else { + console.log("Set hold to " + hold); + } + } +}); + + export const applyAction = (action) => { if (window.__TAURI_INTERNALS__) { invoke("apply_action", { action: action }); diff --git a/game_controller_app/src/handlers.rs b/game_controller_app/src/handlers.rs index ba69735..e347137 100644 --- a/game_controller_app/src/handlers.rs +++ b/game_controller_app/src/handlers.rs @@ -115,6 +115,14 @@ fn declare_actions(actions: Vec, state: State) { let _ = state.subscribed_actions_sender.send(actions); } +#[command] +fn set_mute(mute: bool, state: State) { + println!("Setting mute to {}", mute); + let _ = state.mute_sender.send(mute); +} + +// TODO set_hold + /// This function returns a handler that can be passed to [tauri::Builder::invoke_handler]. /// It must be boxed because otherwise its size is unknown at compile time. pub fn get_invoke_handler() -> Box> { @@ -124,5 +132,6 @@ pub fn get_invoke_handler() -> Box> { get_launch_data, launch, sync_with_backend, + set_mute, ]) } diff --git a/game_controller_core/Cargo.toml b/game_controller_core/Cargo.toml index 0b62884..3fa18da 100644 --- a/game_controller_core/Cargo.toml +++ b/game_controller_core/Cargo.toml @@ -12,4 +12,5 @@ enum-map = { workspace = true } serde = { workspace = true } serde_with = { workspace = true } time = { workspace = true } +tokio = { workspace = true } trait_enum = { workspace = true } diff --git a/game_controller_core/src/action.rs b/game_controller_core/src/action.rs index db779ee..cf21904 100644 --- a/game_controller_core/src/action.rs +++ b/game_controller_core/src/action.rs @@ -21,6 +21,14 @@ pub trait Action { /// This function returns whether the action is legal in the given game state. fn is_legal(&self, c: &ActionContext) -> bool; + + /// This function returns the message to be spoken by the TTS system for speakable actions, + /// or None if the action is not intended to be announced. + fn get_tts_message(&self, _c: &ActionContext) -> Option { + // This default None is so that non-speakable actions don't have to worry + // about this new addition at all. + None + } } trait_enum! { diff --git a/game_controller_core/src/actions/finish_set_play.rs b/game_controller_core/src/actions/finish_set_play.rs index bf35e0b..d39eb1b 100644 --- a/game_controller_core/src/actions/finish_set_play.rs +++ b/game_controller_core/src/actions/finish_set_play.rs @@ -20,4 +20,8 @@ impl Action for FinishSetPlay { fn is_legal(&self, c: &ActionContext) -> bool { c.game.state == State::Playing && c.game.set_play != SetPlay::NoSetPlay } + + fn get_tts_message(&self, _c: &ActionContext) -> Option { + Some("Ball free".to_string()) + } } diff --git a/game_controller_core/src/actions/global_game_stuck.rs b/game_controller_core/src/actions/global_game_stuck.rs index df6397e..996589a 100644 --- a/game_controller_core/src/actions/global_game_stuck.rs +++ b/game_controller_core/src/actions/global_game_stuck.rs @@ -22,4 +22,8 @@ impl Action for GlobalGameStuck { && c.game.state == State::Playing && c.params.competition.challenge_mode.is_none() } + + fn get_tts_message(&self, _c: &ActionContext) -> Option { + Some("Global game stuck".to_string()) + } } diff --git a/game_controller_core/src/actions/goal.rs b/game_controller_core/src/actions/goal.rs index 30443e1..3dbc7cc 100644 --- a/game_controller_core/src/actions/goal.rs +++ b/game_controller_core/src/actions/goal.rs @@ -65,4 +65,11 @@ impl Action for Goal { && (c.params.competition.challenge_mode.is_none() || self.side == c.params.game.kick_off_side) } + + fn get_tts_message(&self, c: &ActionContext) -> Option { + Some(format!( + "Goal for {}", + c.params.game.teams[self.side].field_player_color, + )) + } } diff --git a/game_controller_core/src/actions/penalize.rs b/game_controller_core/src/actions/penalize.rs index 6f90d32..f5da59a 100644 --- a/game_controller_core/src/actions/penalize.rs +++ b/game_controller_core/src/actions/penalize.rs @@ -218,4 +218,17 @@ impl Action for Penalize { } }) } + + fn get_tts_message(&self, c: &ActionContext) -> Option { + let player_number_str = match self.player { + Some(n) => format!("{}", u8::from(n)), + None => "Team".to_string(), + }; + Some(format!( + "{} {} {}", + self.call, + c.params.game.teams[self.side].field_player_color, + player_number_str + )) + } } diff --git a/game_controller_core/src/actions/start_set_play.rs b/game_controller_core/src/actions/start_set_play.rs index 290ac31..9ab3179 100644 --- a/game_controller_core/src/actions/start_set_play.rs +++ b/game_controller_core/src/actions/start_set_play.rs @@ -92,4 +92,24 @@ impl Action for StartSetPlay { && c.params.competition.challenge_mode.is_none() }) } + + fn get_tts_message(&self, c: &ActionContext) -> Option { + let side_color_str = match self.side { + Some(sd) => { + format!("{}", c.params.game.teams[sd].field_player_color) + }, + None => "".to_string(), + }; + let msg = format!( + "{} {}", + self.set_play, + side_color_str, + ); + match self.set_play { + SetPlay::KickIn => Some(msg), + SetPlay::GoalKick => Some(msg), + SetPlay::CornerKick => Some(msg), + _ => None, + } + } } diff --git a/game_controller_core/src/actions/timeout.rs b/game_controller_core/src/actions/timeout.rs index 73e8264..f2a562a 100644 --- a/game_controller_core/src/actions/timeout.rs +++ b/game_controller_core/src/actions/timeout.rs @@ -90,4 +90,13 @@ impl Action for Timeout { && c.game.teams[side].timeout_budget > 0 }) } + + fn get_tts_message(&self, c: &ActionContext) -> Option { + Some( + match self.side { + Some(sd) => format!("Timeout {}", c.params.game.teams[sd].field_player_color), + None => format!("Referee Timeout"), + } + ) + } } diff --git a/game_controller_core/src/actions/unpenalize.rs b/game_controller_core/src/actions/unpenalize.rs index 525ed13..c61a5c6 100644 --- a/game_controller_core/src/actions/unpenalize.rs +++ b/game_controller_core/src/actions/unpenalize.rs @@ -32,4 +32,12 @@ impl Action for Unpenalize { && c.game.state == State::Set) || c.params.game.test.unpenalize) } + + fn get_tts_message(&self, c: &ActionContext) -> Option { + Some(format!( + "{} {} returning to field", + c.params.game.teams[self.side].field_player_color, + u8::from(self.player) + )) + } } diff --git a/game_controller_core/src/lib.rs b/game_controller_core/src/lib.rs index df94395..ded4130 100644 --- a/game_controller_core/src/lib.rs +++ b/game_controller_core/src/lib.rs @@ -15,6 +15,8 @@ use std::{cmp::min, iter::once, time::Duration}; use enum_map::EnumMap; +use tokio::sync::mpsc; + use crate::action::{ActionContext, VAction}; use crate::log::{LogEntry, LoggedAction, Logger, TimestampedLogEntry}; use crate::timer::{BehaviorAtZero, EvaluatedRunConditions, RunCondition, Timer}; @@ -38,11 +40,17 @@ pub struct GameController { time: Duration, history: Vec<(Game, VAction)>, logger: Box, + // to the tts + action_ttsmsg_sender: mpsc::UnboundedSender>, } impl GameController { /// This function creates a new instance with given parameters and a logger. - pub fn new(params: Params, logger: Box) -> Self { + pub fn new( + params: Params, + logger: Box, + action_ttsmsg_sender: mpsc::UnboundedSender> + ) -> Self { let game = Game { sides: params.game.side_mapping, phase: Phase::FirstHalf, @@ -100,6 +108,7 @@ impl GameController { time: Duration::ZERO, history: vec![], logger, + action_ttsmsg_sender, } } @@ -211,6 +220,7 @@ impl GameController { } action.execute(&mut context); + let _ = self.action_ttsmsg_sender.send(action.get_tts_message(&mut context)); // I'm not sure if timer-triggered actions from the non-delayed state should still cancel // the delayed state if they are illegal. if source != ActionSource::Timer { diff --git a/game_controller_core/src/types.rs b/game_controller_core/src/types.rs index ced812a..d63db34 100644 --- a/game_controller_core/src/types.rs +++ b/game_controller_core/src/types.rs @@ -213,6 +213,21 @@ pub enum SetPlay { PenaltyKick, } +impl std::fmt::Display for SetPlay { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let output = match self { + Self::NoSetPlay => "No set play", + Self::KickOff => "Kick-off", + Self::KickIn => "Kick-in", + Self::GoalKick => "Goal kick", + Self::CornerKick => "Corner kick", + Self::PushingFreeKick => "Pushing free kick", + Self::PenaltyKick => "Penalty kick", + }; + write!(f, "{output}") + } +} + /// This enumerates the jersey colors. Values may be added to match actually submitted jersey designs. #[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] @@ -229,6 +244,25 @@ pub enum Color { Gray, } +impl std::fmt::Display for Color { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let output = match self { + Self::Red => "Red", + Self::Blue => "Blue", + Self::Yellow => "Yellow", + Self::Black => "Black", + Self::White => "White", + Self::Green => "Green", + Self::Orange => "Orange", + Self::Purple => "Purple", + Self::Brown => "Brown", + Self::Gray => "Gray", + }; + write!(f, "{output}") + } + +} + /// This enumerates the reasons why a player can be penalized. #[derive(Clone, Copy, Debug, Deserialize, Enum, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] @@ -285,6 +319,27 @@ pub enum PenaltyCall { LeavingTheField, } +impl std::fmt::Display for PenaltyCall { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let output = match self { + Self::RequestForPickUp => "Request for Pickup", + Self::IllegalPosition => "Illegal Position", + Self::MotionInStandby => "Motion in Standby", + Self::MotionInSet => "Motion in Set", + Self::FallenInactive => "Fallen Robot", + Self::LocalGameStuck => "Local Game Stuck", + Self::BallHolding => "Ball Holding", + Self::PlayerStance => "Illegal Stance", + Self::Pushing => "Pushing", + Self::Foul => "Foul", + Self::PenaltyKick => "Penalty Kick", + Self::PlayingWithArmsHands => "Playing with Hands", + Self::LeavingTheField => "Leaving the Field", + }; + write!(f, "{}", output) + } +} + /// This enumerates the two opposing teams of the game. The name `Side` may be slightly misleading /// since it doesn't refer to the side of the field of play, but the order of teams in the /// schedule. A team never changes its home/away designation during a game which is useful for diff --git a/game_controller_runtime/Cargo.toml b/game_controller_runtime/Cargo.toml index c4fecca..a880d78 100644 --- a/game_controller_runtime/Cargo.toml +++ b/game_controller_runtime/Cargo.toml @@ -14,6 +14,7 @@ enum-map = { workspace = true } game_controller_core = { workspace = true } game_controller_msgs = { workspace = true } game_controller_net = { workspace = true } +game_controller_tts = { workspace = true } network-interface = { workspace = true } serde = { workspace = true } serde_with = { workspace = true } diff --git a/game_controller_runtime/src/lib.rs b/game_controller_runtime/src/lib.rs index 1641766..8473b9c 100644 --- a/game_controller_runtime/src/lib.rs +++ b/game_controller_runtime/src/lib.rs @@ -37,6 +37,7 @@ use game_controller_net::{ ControlMessageSender, Event, MonitorRequestReceiver, StatusMessageForwarder, StatusMessageReceiver, TeamMessageReceiver, }; +use game_controller_tts::{tts_event_loop}; pub mod cli; mod connection_status; @@ -88,6 +89,8 @@ pub struct RuntimeState { shutdown_token: CancellationToken, /// The mutable state behind a mutex. It is a tokio mutex because it is held across await. mutable_state: Mutex, + // for the tts settings + pub mute_sender: mpsc::UnboundedSender, } /// This function starts all network services that are not tied to a specific monitor. It returns a @@ -402,7 +405,12 @@ pub async fn start_runtime( .await .context("could not create logger")?; - let mut game_controller = GameController::new(params.clone(), Box::new(logger)); + let (action_ttsmsg_sender, action_ttsmsg_receiver) = mpsc::unbounded_channel(); + let mut game_controller = GameController::new( + params.clone(), + Box::new(logger), + action_ttsmsg_sender, + ); game_controller.log_now(LogEntry::Metadata(LoggedMetadata { creator: "GameController".into(), @@ -474,6 +482,8 @@ pub async fn start_runtime( let ui_notify = Arc::new(Notify::new()); let shutdown_token = CancellationToken::new(); + let (mute_sender, mute_receiver) = mpsc::unbounded_channel(); + runtime_join_set.spawn(event_loop( game_controller, event_receiver, @@ -485,6 +495,12 @@ pub async fn start_runtime( send_ui_state, )); + runtime_join_set.spawn(tts_event_loop( + action_ttsmsg_receiver, + mute_receiver, + shutdown_token.clone(), + )); + Ok(RuntimeState { action_sender, subscribed_actions_sender, @@ -495,6 +511,7 @@ pub async fn start_runtime( runtime_join_set, network_join_set, }), + mute_sender, }) } diff --git a/game_controller_tts/Cargo.toml b/game_controller_tts/Cargo.toml new file mode 100644 index 0000000..9a927dc --- /dev/null +++ b/game_controller_tts/Cargo.toml @@ -0,0 +1,15 @@ +[package] +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +name = "game_controller_tts" +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tts = { version = "0.26.3" } +speech-dispatcher = { version = "0.16.0" } diff --git a/game_controller_tts/src/lib.rs b/game_controller_tts/src/lib.rs new file mode 100644 index 0000000..cbd6e02 --- /dev/null +++ b/game_controller_tts/src/lib.rs @@ -0,0 +1,62 @@ +/** + * Author: Francesco Petri + * + * This file implements the text-to-speech module. + * A whole tokio async module is probably a little overkill for how naive + * and frontend-assisted this ended up being, but still, this modular structure + * should allow a more Rust-savvy contributor greater freedom to apply + * further improvements, as opposed to cramming this in the game_controller_core. + * For example, an explicit message queue we have full control over + * would improve HOLD mode, but handling the callback interface offered + * by the TTS crate proved too tricky for my novice skills. + * Or maybe, as a minor change, one may want to try a different TTS library: + * this is as easy as replacing naive_say with whatever functiion you can write. + */ + +use tts::*; +use tokio::{select, sync::mpsc,}; +use tokio_util::sync::CancellationToken; +use anyhow::Result; + +pub fn naive_say(message: String) { + println!("Saying: {}", message); + let mut the_tts: Tts = Tts::default().unwrap(); // TODO error handling here + let _ = the_tts.speak(message, false); +} + +pub async fn tts_event_loop( + mut action_ttsmsg_receiver: mpsc::UnboundedReceiver>, + mut mute_receiver: mpsc::UnboundedReceiver, + shutdown_token: CancellationToken, +) -> Result<()> { + println!("zan zan zan"); + let mut the_mute = false; + + loop { + select! { + message_or_error = action_ttsmsg_receiver.recv() => { + if let Some(message_or_none) = message_or_error { + if let Some(message) = message_or_none { + println!("Received: {}", message); + if the_mute { + println!("But am mute"); + } + else { + naive_say(message); + } + } + } + }, + mute_or_error = mute_receiver.recv() => { + if let Some(mute) = mute_or_error { + println!("Confirm setting mute to {}", mute); + the_mute = mute; + } + }, + _ = shutdown_token.cancelled() => { + return Ok(()); + }, + }; + } + +} From ea11cbd0cc3f6fac72d45c7aaf2de3a4d3f8a616 Mon Sep 17 00:00:00 2001 From: Francesco Petri Date: Wed, 2 Jul 2025 12:37:01 +0200 Subject: [PATCH 2/3] Add HOLD mode, change version tag HOLD mode will delay all messages while Spacebar is pressed until Spacebar is released. --- Cargo.lock | 14 +++++++------- Cargo.toml | 5 ++++- frontend/src/api.js | 3 +++ game_controller_app/src/handlers.rs | 7 +++++-- game_controller_app/tauri.conf.json | 2 +- game_controller_runtime/src/lib.rs | 4 ++++ game_controller_tts/Cargo.toml | 4 ++-- game_controller_tts/src/lib.rs | 27 +++++++++++++++++++++++++-- 8 files changed, 51 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ba81e2..a0bbd7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1101,7 +1101,7 @@ dependencies = [ [[package]] name = "game_controller_app" -version = "5.0.0-rc.2" +version = "5.0.0-rc.2-tts" dependencies = [ "anyhow", "clap", @@ -1114,7 +1114,7 @@ dependencies = [ [[package]] name = "game_controller_core" -version = "5.0.0-rc.2" +version = "5.0.0-rc.2-tts" dependencies = [ "enum-map", "serde", @@ -1126,7 +1126,7 @@ dependencies = [ [[package]] name = "game_controller_logs" -version = "5.0.0-rc.2" +version = "5.0.0-rc.2-tts" dependencies = [ "anyhow", "bytes", @@ -1139,7 +1139,7 @@ dependencies = [ [[package]] name = "game_controller_msgs" -version = "5.0.0-rc.2" +version = "5.0.0-rc.2-tts" dependencies = [ "anyhow", "bindgen", @@ -1149,7 +1149,7 @@ dependencies = [ [[package]] name = "game_controller_net" -version = "5.0.0-rc.2" +version = "5.0.0-rc.2-tts" dependencies = [ "anyhow", "bytes", @@ -1161,7 +1161,7 @@ dependencies = [ [[package]] name = "game_controller_runtime" -version = "5.0.0-rc.2" +version = "5.0.0-rc.2-tts" dependencies = [ "anyhow", "clap", @@ -1182,7 +1182,7 @@ dependencies = [ [[package]] name = "game_controller_tts" -version = "5.0.0-rc.2" +version = "5.0.0-rc.2-tts" dependencies = [ "anyhow", "speech-dispatcher", diff --git a/Cargo.toml b/Cargo.toml index 20f91c3..77f10f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "game_controller_logs", "game_controller_net", "game_controller_runtime", + "game_controller_tts", ] resolver = "2" @@ -27,12 +28,14 @@ serde_with = { version = "2.3", features = ["base64", "time_0_3"] } serde_repr = { version = "0.1" } serde_yaml = { version = "0.9" } socket2 = { version = "0.5", features = ["all"] } +speech-dispatcher = { version = "0.16.0" } tauri = { version = "2.1", features = [] } tauri-build = { version = "2.0", features = [] } time = { version = "0.3", features = ["formatting", "local-offset", "macros", "serde"] } tokio = { version = "1.0", features = ["fs", "io-util", "macros", "net", "rt", "rt-multi-thread", "sync", "time"] } tokio-util = { version = "0.7" } trait_enum = { version = "0.5" } +tts = { version = "0.26.3" } [workspace.package] authors = ["Arne Hasselbring "] @@ -40,7 +43,7 @@ edition = "2021" license = "MIT" repository = "https://github.com/RoboCup-SPL/GameController3" rust-version = "1.82" -version = "5.0.0-rc.2" +version = "5.0.0-rc.2-tts" [profile.release-dist] inherits = "release" diff --git a/frontend/src/api.js b/frontend/src/api.js index 205af1a..5bbfa42 100644 --- a/frontend/src/api.js +++ b/frontend/src/api.js @@ -216,6 +216,7 @@ let mute = false; let hold = false; document.addEventListener('keydown', function(event) { + if (event.repeat) return; if (event.key == 'm') { mute = !mute; if (window.__TAURI_INTERNALS__) { @@ -225,6 +226,7 @@ document.addEventListener('keydown', function(event) { } } else if (event.key == ' ') { + event.preventDefault(); hold = true; if (window.__TAURI_INTERNALS__) { invoke("set_hold", { hold: hold }); @@ -236,6 +238,7 @@ document.addEventListener('keydown', function(event) { document.addEventListener('keyup', function(event) { if (event.key == ' ') { + event.preventDefault(); hold = false; if (window.__TAURI_INTERNALS__) { invoke("set_hold", { hold: hold }); diff --git a/game_controller_app/src/handlers.rs b/game_controller_app/src/handlers.rs index e347137..478e0ea 100644 --- a/game_controller_app/src/handlers.rs +++ b/game_controller_app/src/handlers.rs @@ -117,11 +117,13 @@ fn declare_actions(actions: Vec, state: State) { #[command] fn set_mute(mute: bool, state: State) { - println!("Setting mute to {}", mute); let _ = state.mute_sender.send(mute); } -// TODO set_hold +#[command] +fn set_hold(hold: bool, state: State) { + let _ = state.hold_sender.send(hold); +} /// This function returns a handler that can be passed to [tauri::Builder::invoke_handler]. /// It must be boxed because otherwise its size is unknown at compile time. @@ -133,5 +135,6 @@ pub fn get_invoke_handler() -> Box> { launch, sync_with_backend, set_mute, + set_hold, ]) } diff --git a/game_controller_app/tauri.conf.json b/game_controller_app/tauri.conf.json index 355d330..fcac305 100644 --- a/game_controller_app/tauri.conf.json +++ b/game_controller_app/tauri.conf.json @@ -48,5 +48,5 @@ "identifier": "org.robocup.spl.game-controller", "mainBinaryName": "GameController", "productName": "GameController", - "version": "5.0.0-rc.2" + "version": "5.0.0-rc.2-tts" } diff --git a/game_controller_runtime/src/lib.rs b/game_controller_runtime/src/lib.rs index 8473b9c..45b9292 100644 --- a/game_controller_runtime/src/lib.rs +++ b/game_controller_runtime/src/lib.rs @@ -91,6 +91,7 @@ pub struct RuntimeState { mutable_state: Mutex, // for the tts settings pub mute_sender: mpsc::UnboundedSender, + pub hold_sender: mpsc::UnboundedSender, } /// This function starts all network services that are not tied to a specific monitor. It returns a @@ -483,6 +484,7 @@ pub async fn start_runtime( let shutdown_token = CancellationToken::new(); let (mute_sender, mute_receiver) = mpsc::unbounded_channel(); + let (hold_sender, hold_receiver) = mpsc::unbounded_channel(); runtime_join_set.spawn(event_loop( game_controller, @@ -498,6 +500,7 @@ pub async fn start_runtime( runtime_join_set.spawn(tts_event_loop( action_ttsmsg_receiver, mute_receiver, + hold_receiver, shutdown_token.clone(), )); @@ -512,6 +515,7 @@ pub async fn start_runtime( network_join_set, }), mute_sender, + hold_sender, }) } diff --git a/game_controller_tts/Cargo.toml b/game_controller_tts/Cargo.toml index 9a927dc..8ad9a3c 100644 --- a/game_controller_tts/Cargo.toml +++ b/game_controller_tts/Cargo.toml @@ -11,5 +11,5 @@ version = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } -tts = { version = "0.26.3" } -speech-dispatcher = { version = "0.16.0" } +tts = { workspace = true } +speech-dispatcher = { workspace = true } diff --git a/game_controller_tts/src/lib.rs b/game_controller_tts/src/lib.rs index cbd6e02..83ebff6 100644 --- a/game_controller_tts/src/lib.rs +++ b/game_controller_tts/src/lib.rs @@ -27,10 +27,12 @@ pub fn naive_say(message: String) { pub async fn tts_event_loop( mut action_ttsmsg_receiver: mpsc::UnboundedReceiver>, mut mute_receiver: mpsc::UnboundedReceiver, + mut hold_receiver: mpsc::UnboundedReceiver, shutdown_token: CancellationToken, ) -> Result<()> { - println!("zan zan zan"); let mut the_mute = false; + let mut the_hold = false; + let mut held_messages = vec![]; loop { select! { @@ -41,6 +43,10 @@ pub async fn tts_event_loop( if the_mute { println!("But am mute"); } + else if the_hold { + println!("Holding it"); + held_messages.push(message); + } else { naive_say(message); } @@ -49,8 +55,25 @@ pub async fn tts_event_loop( }, mute_or_error = mute_receiver.recv() => { if let Some(mute) = mute_or_error { - println!("Confirm setting mute to {}", mute); + println!("Set mute to {}", mute); the_mute = mute; + if mute { + // have mute double as "hold-cancel" + held_messages.clear(); + } + } + }, + hold_or_error = hold_receiver.recv() => { + if let Some(hold) = hold_or_error { + println!("Set hold to {}", hold); + the_hold = hold; + if !hold { + // release all held messages + for i in 0..held_messages.len() { + naive_say(held_messages[i].clone()); + } + held_messages.clear(); + } } }, _ = shutdown_token.cancelled() => { From 2ed60fe928ba5b2cb06fad95e3a7bb6135a71e0a Mon Sep 17 00:00:00 2001 From: Francesco Petri Date: Wed, 2 Jul 2025 17:18:42 +0200 Subject: [PATCH 3/3] Add TTS settings --- frontend/src/components/Launcher.jsx | 13 +- .../src/components/launcher/TtsSettings.jsx | 44 ++++++ game_controller_runtime/src/launch.rs | 23 +++ game_controller_runtime/src/lib.rs | 2 + game_controller_tts/src/lib.rs | 133 +++++++++++------- 5 files changed, 165 insertions(+), 50 deletions(-) create mode 100644 frontend/src/components/launcher/TtsSettings.jsx diff --git a/frontend/src/components/Launcher.jsx b/frontend/src/components/Launcher.jsx index 2e268eb..23ed9e6 100644 --- a/frontend/src/components/Launcher.jsx +++ b/frontend/src/components/Launcher.jsx @@ -3,6 +3,7 @@ import CompetitionSettings from "./launcher/CompetitionSettings"; import GameSettings from "./launcher/GameSettings"; import NetworkSettings from "./launcher/NetworkSettings"; import WindowSettings from "./launcher/WindowSettings"; +import TtsSettings from "./launcher/TtsSettings"; import { getLaunchData, launch } from "../api"; const Launcher = ({ setLaunched }) => { @@ -10,6 +11,7 @@ const Launcher = ({ setLaunched }) => { const [launchSettings, setLaunchSettings] = useState(null); const [networkInterfaces, setNetworkInterfaces] = useState(null); const [teams, setTeams] = useState(null); + const [voices, setVoices] = useState(null); const launchSettingsAreLegal = launchSettings != null && @@ -31,6 +33,7 @@ const Launcher = ({ setLaunched }) => { setLaunchSettings(data.defaultSettings); setNetworkInterfaces(data.networkInterfaces); setTeams(data.teams); + setVoices(data.voices); }); }, []); @@ -38,7 +41,8 @@ const Launcher = ({ setLaunched }) => { competitions != null && launchSettings != null && networkInterfaces != null && - teams != null + teams != null && + voices != null ) { const setCompetition = (competition) => { const INVISIBLES_NUMBER = 0; @@ -69,6 +73,7 @@ const Launcher = ({ setLaunched }) => { const teamsInThisCompetition = teams.filter((team) => thisCompetition.teams.includes(team.number) ); + const languages = Object.keys(voices).sort() return (
{ network={launchSettings.network} setNetwork={(network) => setLaunchSettings({ ...launchSettings, network: network })} /> + setLaunchSettings({ ...launchSettings, tts: tts })} + />