From 917b6a8fcb2fef31e50ea5caf759765a23cefe6a Mon Sep 17 00:00:00 2001 From: pham-tuan-binh Date: Thu, 12 Mar 2026 13:30:24 +0700 Subject: [PATCH 1/7] Add livekit-wakeword as workspace dependency and wire into livekit-uniffi --- Cargo.lock | 2 ++ Cargo.toml | 1 + livekit-uniffi/Cargo.toml | 2 ++ livekit-uniffi/src/lib.rs | 3 +++ 4 files changed, 8 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 0f005f9d4..cb1810bc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3949,8 +3949,10 @@ version = "0.1.0" dependencies = [ "livekit-api", "livekit-protocol", + "livekit-wakeword", "log", "once_cell", + "thiserror 1.0.69", "tokio", "uniffi", ] diff --git a/Cargo.toml b/Cargo.toml index ecf593d5d..936bd7f0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ livekit-api = { version = "0.4.14", path = "livekit-api" } livekit-ffi = { version = "0.12.48", path = "livekit-ffi" } livekit-protocol = { version = "0.7.1", path = "livekit-protocol" } livekit-runtime = { version = "0.4.0", path = "livekit-runtime" } +livekit-wakeword = { version = "0.1.0", path = "livekit-wakeword" } soxr-sys = { version = "0.1.2", path = "soxr-sys" } webrtc-sys = { version = "0.3.23", path = "webrtc-sys" } webrtc-sys-build = { version = "0.3.13", path = "webrtc-sys/build" } diff --git a/livekit-uniffi/Cargo.toml b/livekit-uniffi/Cargo.toml index 8293512e9..4170b3509 100644 --- a/livekit-uniffi/Cargo.toml +++ b/livekit-uniffi/Cargo.toml @@ -14,8 +14,10 @@ publish = false [dependencies] livekit-protocol = { workspace = true } livekit-api = { workspace = true } +livekit-wakeword = { workspace = true } uniffi = { version = "0.30.0", features = ["cli", "scaffolding-ffi-buffer-fns"] } log = { workspace = true } +thiserror = { workspace = true } tokio = { workspace = true, features = ["sync"] } once_cell = "1.21.3" diff --git a/livekit-uniffi/src/lib.rs b/livekit-uniffi/src/lib.rs index 698bd4d0b..d633d4203 100644 --- a/livekit-uniffi/src/lib.rs +++ b/livekit-uniffi/src/lib.rs @@ -21,4 +21,7 @@ pub mod log_forward; /// Information about the build such as version. pub mod build_info; +/// Wake word detection from [`livekit-wakeword`]. +pub mod wakeword; + uniffi::setup_scaffolding!(); From a2b017a62465745d1ef5b4f843634540e52ad1e3 Mon Sep 17 00:00:00 2001 From: pham-tuan-binh Date: Thu, 12 Mar 2026 13:30:38 +0700 Subject: [PATCH 2/7] Add UniFFI interface for livekit-wakeword Expose WakeWordDetector as a UniFFI Object wrapping WakeWordModel with Mutex for interior mutability. Includes a flat wrapper error type that converts from WakeWordError since its inner types (ort::Error, ndarray::ShapeError, etc.) are not UniFFI-compatible. --- livekit-uniffi/src/wakeword.rs | 92 ++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 livekit-uniffi/src/wakeword.rs diff --git a/livekit-uniffi/src/wakeword.rs b/livekit-uniffi/src/wakeword.rs new file mode 100644 index 000000000..f45104a30 --- /dev/null +++ b/livekit-uniffi/src/wakeword.rs @@ -0,0 +1,92 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use livekit_wakeword::{WakeWordError as InnerWakeWordError, WakeWordModel}; +use std::collections::HashMap; +use std::sync::Mutex; + +/// An error that can occur during wake word detection. +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum WakeWordError { + #[error("{msg}")] + ModelNotFound { msg: String }, + #[error("{msg}")] + UnsupportedSampleRate { msg: String }, + #[error("{msg}")] + IoError { msg: String }, + #[error("{msg}")] + InferenceError { msg: String }, +} + +impl From for WakeWordError { + fn from(e: InnerWakeWordError) -> Self { + match &e { + InnerWakeWordError::ModelNotFound(_) => { + WakeWordError::ModelNotFound { msg: e.to_string() } + } + InnerWakeWordError::UnsupportedSampleRate(_) => { + WakeWordError::UnsupportedSampleRate { msg: e.to_string() } + } + InnerWakeWordError::Io(_) => WakeWordError::IoError { msg: e.to_string() }, + _ => WakeWordError::InferenceError { msg: e.to_string() }, + } + } +} + +/// Wake word detector backed by ONNX classifier models. +/// +/// Wraps [`livekit_wakeword::WakeWordModel`] for use across FFI boundaries. +/// Uses interior mutability since the underlying model requires `&mut self`. +#[derive(uniffi::Object)] +pub struct WakeWordDetector { + inner: Mutex, +} + +#[uniffi::export] +impl WakeWordDetector { + /// Create a new wake word detector. + /// + /// `model_paths` are filesystem paths to ONNX classifier models. + /// `sample_rate` is the sample rate of audio that will be passed to + /// [`predict`](Self::predict). Supported rates: 16000 (recommended), + /// 22050, 32000, 44100, 48000, 88200, 96000, 176400, 192000, 384000 Hz. + #[uniffi::constructor] + pub fn new(model_paths: Vec, sample_rate: u32) -> Result { + let model = WakeWordModel::new(&model_paths, sample_rate)?; + Ok(Self { inner: Mutex::new(model) }) + } + + /// Load an additional wake word classifier ONNX model from disk. + /// + /// If `model_name` is `None`, the file stem is used as the classifier name. + pub fn load_model( + &self, + model_path: String, + model_name: Option, + ) -> Result<(), WakeWordError> { + let mut inner = self.inner.lock().unwrap(); + inner.load_model(&model_path, model_name.as_deref())?; + Ok(()) + } + + /// Get wake word predictions for an audio chunk. + /// + /// Pass ~2 seconds of i16 PCM audio at the sample rate configured in + /// [`new`](Self::new). Returns a map of classifier name to confidence score. + pub fn predict(&self, audio_chunk: Vec) -> Result, WakeWordError> { + let mut inner = self.inner.lock().unwrap(); + Ok(inner.predict(&audio_chunk)?) + } +} From ca7101fae843569dd7dfbf653c1c6491df127d8d Mon Sep 17 00:00:00 2001 From: pham-tuan-binh Date: Thu, 12 Mar 2026 13:33:34 +0700 Subject: [PATCH 3/7] Use uniffi::remote(Error) for WakeWordError instead of wrapper type Drop the thiserror wrapper error and From impl in favor of #[uniffi::remote(Error)] with #[uniffi(flat_error)], matching the pattern used by AccessTokenError. Removes the thiserror dependency from livekit-uniffi. --- livekit-uniffi/Cargo.toml | 1 - livekit-uniffi/src/wakeword.rs | 34 ++++++++-------------------------- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/livekit-uniffi/Cargo.toml b/livekit-uniffi/Cargo.toml index 4170b3509..62dfaa637 100644 --- a/livekit-uniffi/Cargo.toml +++ b/livekit-uniffi/Cargo.toml @@ -17,7 +17,6 @@ livekit-api = { workspace = true } livekit-wakeword = { workspace = true } uniffi = { version = "0.30.0", features = ["cli", "scaffolding-ffi-buffer-fns"] } log = { workspace = true } -thiserror = { workspace = true } tokio = { workspace = true, features = ["sync"] } once_cell = "1.21.3" diff --git a/livekit-uniffi/src/wakeword.rs b/livekit-uniffi/src/wakeword.rs index f45104a30..05e5a9d75 100644 --- a/livekit-uniffi/src/wakeword.rs +++ b/livekit-uniffi/src/wakeword.rs @@ -12,37 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use livekit_wakeword::{WakeWordError as InnerWakeWordError, WakeWordModel}; +use livekit_wakeword::{WakeWordError, WakeWordModel}; use std::collections::HashMap; use std::sync::Mutex; -/// An error that can occur during wake word detection. -#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi::remote(Error)] #[uniffi(flat_error)] pub enum WakeWordError { - #[error("{msg}")] - ModelNotFound { msg: String }, - #[error("{msg}")] - UnsupportedSampleRate { msg: String }, - #[error("{msg}")] - IoError { msg: String }, - #[error("{msg}")] - InferenceError { msg: String }, -} - -impl From for WakeWordError { - fn from(e: InnerWakeWordError) -> Self { - match &e { - InnerWakeWordError::ModelNotFound(_) => { - WakeWordError::ModelNotFound { msg: e.to_string() } - } - InnerWakeWordError::UnsupportedSampleRate(_) => { - WakeWordError::UnsupportedSampleRate { msg: e.to_string() } - } - InnerWakeWordError::Io(_) => WakeWordError::IoError { msg: e.to_string() }, - _ => WakeWordError::InferenceError { msg: e.to_string() }, - } - } + Ort, + Shape, + Io, + ModelNotFound, + UnsupportedSampleRate, + Resample, } /// Wake word detector backed by ONNX classifier models. From d6b1cd4614c8dab724a3dfb619e0931e81afc2ae Mon Sep 17 00:00:00 2001 From: pham-tuan-binh Date: Thu, 12 Mar 2026 13:35:13 +0700 Subject: [PATCH 4/7] Add changeset for livekit-wakeword UniFFI interface --- .../add_uniffi_interface_for_livekit_wakeword.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/add_uniffi_interface_for_livekit_wakeword.md diff --git a/.changeset/add_uniffi_interface_for_livekit_wakeword.md b/.changeset/add_uniffi_interface_for_livekit_wakeword.md new file mode 100644 index 000000000..3b2e20ed8 --- /dev/null +++ b/.changeset/add_uniffi_interface_for_livekit_wakeword.md @@ -0,0 +1,10 @@ +--- +livekit-uniffi: minor +--- + +# Add UniFFI interface for livekit-wakeword + +## Summary +- Expose `WakeWordDetector` as a UniFFI Object wrapping `WakeWordModel` with `Mutex` for interior mutability +- Export `new`, `load_model`, and `predict` methods across FFI +- Use `#[uniffi::remote(Error)]` with `#[uniffi(flat_error)]` for `WakeWordError`, matching the existing `AccessTokenError` pattern From c39355c0199c629609e0d35832b401889eb540a2 Mon Sep 17 00:00:00 2001 From: pham-tuan-binh Date: Thu, 12 Mar 2026 22:04:53 +0700 Subject: [PATCH 5/7] Add download-binaries and tls-native features to ort dependency Fixes Windows CI linking error by enabling ort to download prebuilt ONNX Runtime binaries on all platforms. --- Cargo.lock | 69 ++++++++++++++++++++++++++++++++++++- livekit-wakeword/Cargo.toml | 2 +- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb1810bc5..f6368c094 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2972,6 +2972,12 @@ dependencies = [ "digest", ] +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + [[package]] name = "home" version = "0.5.12" @@ -3952,7 +3958,6 @@ dependencies = [ "livekit-wakeword", "log", "once_cell", - "thiserror 1.0.69", "tokio", "uniffi", ] @@ -4045,6 +4050,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lzma-rust2" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1670343e58806300d87950e3401e820b519b9384281bbabfb15e3636689ffd69" + [[package]] name = "mach2" version = "0.4.3" @@ -5105,6 +5116,7 @@ dependencies = [ "ort-sys", "smallvec", "tracing", + "ureq", ] [[package]] @@ -5112,6 +5124,11 @@ name = "ort-sys" version = "2.0.0-rc.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06503bb33f294c5f1ba484011e053bfa6ae227074bdb841e9863492dc5960d4b" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] [[package]] name = "ort-tract" @@ -6823,6 +6840,17 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "soxr-sys" version = "0.1.2" @@ -7907,6 +7935,36 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc97a28575b85cfedf2a7e7d3cc64b3e11bd8ac766666318003abbacc7a21fc" +dependencies = [ + "base64 0.22.1", + "der", + "log", + "native-tls", + "percent-encoding", + "rustls-pki-types", + "socks", + "ureq-proto", + "utf-8", + "webpki-root-certs", +] + +[[package]] +name = "ureq-proto" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f" +dependencies = [ + "base64 0.22.1", + "http 1.4.0", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -8277,6 +8335,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.25.4" diff --git a/livekit-wakeword/Cargo.toml b/livekit-wakeword/Cargo.toml index a716e4765..4bd6b9ea4 100644 --- a/livekit-wakeword/Cargo.toml +++ b/livekit-wakeword/Cargo.toml @@ -7,7 +7,7 @@ license.workspace = true [dependencies] ndarray = "0.17.2" -ort = { version = "2.0.0-rc.11", default-features = false, features = ["ndarray", "std"] } +ort = { version = "2.0.0-rc.11", default-features = false, features = ["ndarray", "std", "download-binaries", "tls-native"] } resampler = "0.4" thiserror = "2" From 08d2d58f38c17a3fa5b34509af417bd1ab006b2e Mon Sep 17 00:00:00 2001 From: pham-tuan-binh Date: Thu, 12 Mar 2026 22:45:55 +0700 Subject: [PATCH 6/7] Remove crt-static for aarch64-pc-windows-msvc to fix ort linking The prebuilt ONNX Runtime binaries from ort's download-binaries feature are built with dynamic CRT (/MD), which conflicts with +crt-static (/MT) causing unresolved symbols for CRT functions. --- .cargo/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 2dc4a5047..6d9b1f650 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,7 +2,7 @@ rustflags = ["-C", "target-feature=+crt-static"] [target.aarch64-pc-windows-msvc] -rustflags = ["-C", "target-feature=+crt-static"] +rustflags = [] [target.x86_64-apple-darwin] rustflags = ["-C", "link-args=-ObjC"] From 5fbcb4ee457651ff71bd72d07d23d1d8695510cb Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:59:44 -0700 Subject: [PATCH 7/7] Wakeword support behind feature --- livekit-uniffi/Cargo.toml | 6 +++++- livekit-uniffi/src/lib.rs | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/livekit-uniffi/Cargo.toml b/livekit-uniffi/Cargo.toml index 62dfaa637..899ce19d0 100644 --- a/livekit-uniffi/Cargo.toml +++ b/livekit-uniffi/Cargo.toml @@ -11,10 +11,14 @@ repository.workspace = true readme = "README.md" publish = false +[features] +default = ["wakeword"] +wakeword = ["livekit-wakeword"] + [dependencies] livekit-protocol = { workspace = true } livekit-api = { workspace = true } -livekit-wakeword = { workspace = true } +livekit-wakeword = { workspace = true, optional = true } uniffi = { version = "0.30.0", features = ["cli", "scaffolding-ffi-buffer-fns"] } log = { workspace = true } tokio = { workspace = true, features = ["sync"] } diff --git a/livekit-uniffi/src/lib.rs b/livekit-uniffi/src/lib.rs index d633d4203..eddf55631 100644 --- a/livekit-uniffi/src/lib.rs +++ b/livekit-uniffi/src/lib.rs @@ -22,6 +22,7 @@ pub mod log_forward; pub mod build_info; /// Wake word detection from [`livekit-wakeword`]. +#[cfg(feature = "wakeword")] pub mod wakeword; uniffi::setup_scaffolding!();