Audio output tts remake

Cargo.toml

@@ -7,6 +7,7 @@ members = [
   "game_controller_logs",
   "game_controller_net",
   "game_controller_runtime",
+  "game_controller_tts",
 ]
 resolver = "2"
 
@@ -20,26 +21,29 @@ game_controller_core = { path = "game_controller_core" }
 game_controller_msgs = { path = "game_controller_msgs" }
 game_controller_net = { path = "game_controller_net" }
 game_controller_runtime = { path = "game_controller_runtime" }
+game_controller_tts = { path = "game_controller_tts" }
 network-interface = { version = "1" }
 serde = { version = "1.0", features = ["derive"] }
 serde_with = { version = "2.3", features = ["base64", "time_0_3"] }
 serde_repr = { version = "0.1" }
 serde_yaml = { version = "0.9" }
 socket2 = { version = "0.5", features = ["all"] }
+speech-dispatcher = { version = "0.16.0" }
 tauri = { version = "2.1", features = [] }
 tauri-build = { version = "2.0", features = [] }
 time = { version = "0.3", features = ["formatting", "local-offset", "macros", "serde"] }
 tokio = { version = "1.0", features = ["fs", "io-util", "macros", "net", "rt", "rt-multi-thread", "sync", "time"] }
 tokio-util = { version = "0.7" }
 trait_enum = { version = "0.5" }
+tts = { version = "0.26.3" }
 
 [workspace.package]
 authors = ["Arne Hasselbring <arha@uni-bremen.de>"]
 edition = "2021"
 license = "MIT"
 repository = "https://github.com/RoboCup-SPL/GameController3"
 rust-version = "1.82"
-version = "5.0.0-rc.2"
+version = "5.0.0-rc.2-tts"
 
 [profile.release-dist]
 inherits = "release"

frontend/src/api.js

@@ -210,6 +210,45 @@ export const syncWithBackend = async () => {
   }
 };
 
+// press M to toggle mute mode
+// hold spacebar for hold mode
+let mute = false;
+let hold = false;
+
+document.addEventListener('keydown', function(event) {
+  if (event.repeat) return;
+  if (event.key == 'm') {
+    mute = !mute;
+    if (window.__TAURI_INTERNALS__) {
+      invoke("set_mute", { mute: mute });
+    } else {
+      console.log("Set mute to " + mute);
+    }
+  }
+  else if (event.key == ' ') {
+    event.preventDefault();
+    hold = true;
+    if (window.__TAURI_INTERNALS__) {
+      invoke("set_hold", { hold: hold });
+    } else {
+      console.log("Set hold to " + hold);
+    }
+  }
+});
+
+document.addEventListener('keyup', function(event) {
+  if (event.key == ' ') {
+    event.preventDefault();
+    hold = false;
+    if (window.__TAURI_INTERNALS__) {
+      invoke("set_hold", { hold: hold });
+    } else {
+      console.log("Set hold to " + hold);
+    }
+  }
+});
+
+
 export const applyAction = (action) => {
   if (window.__TAURI_INTERNALS__) {
     invoke("apply_action", { action: action });

frontend/src/components/Launcher.jsx

@@ -3,13 +3,15 @@ import CompetitionSettings from "./launcher/CompetitionSettings";
 import GameSettings from "./launcher/GameSettings";
 import NetworkSettings from "./launcher/NetworkSettings";
 import WindowSettings from "./launcher/WindowSettings";
+import TtsSettings from "./launcher/TtsSettings";
 import { getLaunchData, launch } from "../api";
 
 const Launcher = ({ setLaunched }) => {
   const [competitions, setCompetitions] = useState(null);
   const [launchSettings, setLaunchSettings] = useState(null);
   const [networkInterfaces, setNetworkInterfaces] = useState(null);
   const [teams, setTeams] = useState(null);
+  const [voices, setVoices] = useState(null);
 
   const launchSettingsAreLegal =
     launchSettings != null &&
@@ -31,14 +33,16 @@ const Launcher = ({ setLaunched }) => {
       setLaunchSettings(data.defaultSettings);
       setNetworkInterfaces(data.networkInterfaces);
       setTeams(data.teams);
+      setVoices(data.voices);
     });
   }, []);
 
   if (
     competitions != null &&
     launchSettings != null &&
     networkInterfaces != null &&
-    teams != null
+    teams != null &&
+    voices != null
   ) {
     const setCompetition = (competition) => {
       const INVISIBLES_NUMBER = 0;
@@ -69,6 +73,7 @@ const Launcher = ({ setLaunched }) => {
     const teamsInThisCompetition = teams.filter((team) =>
       thisCompetition.teams.includes(team.number)
     );
+    const languages = Object.keys(voices).sort()
     return (
       <div className="flex flex-col items-center p-4 gap-2">
         <CompetitionSettings
@@ -90,6 +95,12 @@ const Launcher = ({ setLaunched }) => {
           network={launchSettings.network}
           setNetwork={(network) => setLaunchSettings({ ...launchSettings, network: network })}
         />
+        <TtsSettings
+          languages={languages}
+          voices={voices}
+          tts={launchSettings.tts}
+          setTts={(tts) => setLaunchSettings({ ...launchSettings, tts: tts })}
+        />
         <button
           className="px-8 py-2 rounded-md border border-black disabled:bg-slate-400"
           disabled={!launchSettingsAreLegal}

frontend/src/components/launcher/TtsSettings.jsx

@@ -0,0 +1,44 @@
+import { useState } from "react";
+
+const TtsSettings = ({ languages, voices, tts, setTts }) => {
+  const [the_language, set_the_language] = useState("en-US");
+  return (
+    <div className="flex flex-col items-center gap-2">
+      <div className="flex flex-row items-center gap-2">
+        <label>TTS</label>
+        <input
+          type="checkbox"
+          checked={tts.enabled}
+          id="ttsenable"
+          onChange={(e) => setTts({ ...tts, enabled: e.target.checked })}
+        />
+        <label htmlFor="language">language</label>
+        <select
+          id="language"
+          value={the_language}
+          onChange={(e) => {set_the_language(e.target.value); setTts({ ...tts, voice: voices[e.target.value][0]})}}
+        >
+          {languages.map((lang) => (
+            <option key={lang} value={lang}>
+              {lang}
+            </option>
+          ))}
+        </select>
+        <label htmlFor="voice">voice</label>
+        <select
+          id="voice"
+          value={tts.voice}
+          onChange={(e) => setTts({ ...tts, voice: e.target.value })}
+        >
+          {voices[the_language].map((voice) => (
+            <option key={voice} value={voice}>
+              {voice}
+            </option>
+          ))}
+        </select>
+      </div>
+    </div>
+  );
+};
+
+export default TtsSettings;

game_controller_app/src/handlers.rs

@@ -115,6 +115,16 @@ fn declare_actions(actions: Vec<VAction>, state: State<RuntimeState>) {
     let _ = state.subscribed_actions_sender.send(actions);
 }
 
+#[command]
+fn set_mute(mute: bool, state: State<RuntimeState>) {
+    let _ = state.mute_sender.send(mute);
+}
+
+#[command]
+fn set_hold(hold: bool, state: State<RuntimeState>) {
+    let _ = state.hold_sender.send(hold);
+}
+
 /// This function returns a handler that can be passed to [tauri::Builder::invoke_handler].
 /// It must be boxed because otherwise its size is unknown at compile time.
 pub fn get_invoke_handler() -> Box<InvokeHandler<Wry>> {
@@ -124,5 +134,7 @@ pub fn get_invoke_handler() -> Box<InvokeHandler<Wry>> {
         get_launch_data,
         launch,
         sync_with_backend,
+        set_mute,
+        set_hold,
     ])
 }

game_controller_app/tauri.conf.json

@@ -48,5 +48,5 @@
   "identifier": "org.robocup.spl.game-controller",
   "mainBinaryName": "GameController",
   "productName": "GameController",
-  "version": "5.0.0-rc.2"
+  "version": "5.0.0-rc.2-tts"
 }

game_controller_core/Cargo.toml

@@ -12,4 +12,5 @@ enum-map = { workspace = true }
 serde = { workspace = true }
 serde_with = { workspace = true }
 time = { workspace = true }
+tokio = { workspace = true }
 trait_enum = { workspace = true }

game_controller_core/src/action.rs

@@ -21,6 +21,14 @@ pub trait Action {
 
     /// This function returns whether the action is legal in the given game state.
     fn is_legal(&self, c: &ActionContext) -> bool;
+
+    /// This function returns the message to be spoken by the TTS system for speakable actions,
+    /// or None if the action is not intended to be announced.
+    fn get_tts_message(&self, _c: &ActionContext) -> Option<String> {
+        // This default None is so that non-speakable actions don't have to worry
+        // about this new addition at all.
+        None
+    }
 }
 
 trait_enum! {

game_controller_core/src/actions/finish_set_play.rs

@@ -20,4 +20,8 @@ impl Action for FinishSetPlay {
     fn is_legal(&self, c: &ActionContext) -> bool {
         c.game.state == State::Playing && c.game.set_play != SetPlay::NoSetPlay
     }
+
+    fn get_tts_message(&self, _c: &ActionContext) -> Option<String> {
+        Some("Ball free".to_string())
+    }
 }

game_controller_core/src/actions/global_game_stuck.rs

@@ -22,4 +22,8 @@ impl Action for GlobalGameStuck {
             && c.game.state == State::Playing
             && c.params.competition.challenge_mode.is_none()
     }
+
+    fn get_tts_message(&self, _c: &ActionContext) -> Option<String> {
+        Some("Global game stuck".to_string())
+    }
 }

game_controller_core/src/actions/goal.rs

@@ -65,4 +65,11 @@ impl Action for Goal {
             && (c.params.competition.challenge_mode.is_none()
                 || self.side == c.params.game.kick_off_side)
     }
+
+    fn get_tts_message(&self, c: &ActionContext) -> Option<String> {
+        Some(format!(
+            "Goal for {}",
+            c.params.game.teams[self.side].field_player_color,
+        ))
+    }
 }

game_controller_core/src/actions/penalize.rs

@@ -218,4 +218,17 @@ impl Action for Penalize {
                 }
             })
     }
+
+    fn get_tts_message(&self, c: &ActionContext) -> Option<String> {
+        let player_number_str = match self.player {
+            Some(n) => format!("{}", u8::from(n)),
+            None => "Team".to_string(),
+        };
+        Some(format!(
+            "{} {} {}",
+            self.call,
+            c.params.game.teams[self.side].field_player_color,
+            player_number_str
+        ))
+    }
 }

game_controller_core/src/actions/start_set_play.rs

@@ -92,4 +92,24 @@ impl Action for StartSetPlay {
                     && c.params.competition.challenge_mode.is_none()
             })
     }
+
+    fn get_tts_message(&self, c: &ActionContext) -> Option<String> {
+        let side_color_str = match self.side {
+            Some(sd) => {
+                format!("{}", c.params.game.teams[sd].field_player_color)
+            },
+            None => "".to_string(),
+        };
+        let msg = format!(
+            "{} {}",
+            self.set_play,
+            side_color_str,
+        );
+        match self.set_play {
+            SetPlay::KickIn => Some(msg),
+            SetPlay::GoalKick => Some(msg),
+            SetPlay::CornerKick => Some(msg),
+            _ => None,
+        }
+    }
 }

game_controller_core/src/actions/timeout.rs

@@ -90,4 +90,13 @@ impl Action for Timeout {
                     && c.game.teams[side].timeout_budget > 0
             })
     }
+
+    fn get_tts_message(&self, c: &ActionContext) -> Option<String> {
+        Some(
+            match self.side {
+                Some(sd) => format!("Timeout {}", c.params.game.teams[sd].field_player_color),
+                None => format!("Referee Timeout"),
+            }
+        )
+    }
 }

game_controller_core/src/actions/unpenalize.rs

@@ -32,4 +32,12 @@ impl Action for Unpenalize {
                     && c.game.state == State::Set)
                 || c.params.game.test.unpenalize)
     }
+
+    fn get_tts_message(&self, c: &ActionContext) -> Option<String> {
+        Some(format!(
+            "{} {} returning to field",
+            c.params.game.teams[self.side].field_player_color,
+            u8::from(self.player)
+        ))
+    }
 }

game_controller_core/src/lib.rs

@@ -15,6 +15,8 @@ use std::{cmp::min, iter::once, time::Duration};
 
 use enum_map::EnumMap;
 
+use tokio::sync::mpsc;
+
 use crate::action::{ActionContext, VAction};
 use crate::log::{LogEntry, LoggedAction, Logger, TimestampedLogEntry};
 use crate::timer::{BehaviorAtZero, EvaluatedRunConditions, RunCondition, Timer};
@@ -38,11 +40,17 @@ pub struct GameController {
     time: Duration,
     history: Vec<(Game, VAction)>,
     logger: Box<dyn Logger + Send>,
+    // to the tts
+    action_ttsmsg_sender: mpsc::UnboundedSender<Option<String>>,
 }
 
 impl GameController {
     /// This function creates a new instance with given parameters and a logger.
-    pub fn new(params: Params, logger: Box<dyn Logger + Send>) -> Self {
+    pub fn new(
+        params: Params,
+        logger: Box<dyn Logger + Send>,
+        action_ttsmsg_sender: mpsc::UnboundedSender<Option<String>>
+    ) -> Self {
         let game = Game {
             sides: params.game.side_mapping,
             phase: Phase::FirstHalf,
@@ -100,6 +108,7 @@ impl GameController {
             time: Duration::ZERO,
             history: vec![],
             logger,
+            action_ttsmsg_sender,
         }
     }
 
@@ -211,6 +220,7 @@ impl GameController {
         }
 
         action.execute(&mut context);
+        let _ = self.action_ttsmsg_sender.send(action.get_tts_message(&mut context));
         // I'm not sure if timer-triggered actions from the non-delayed state should still cancel
         // the delayed state if they are illegal.
         if source != ActionSource::Timer {