From 85dd1b9b236fe12145c62d764d06b76ef6ea5e2b Mon Sep 17 00:00:00 2001 From: Marcus Date: Tue, 23 Jul 2024 19:28:33 +0200 Subject: [PATCH 01/10] init wayland --- Cargo.toml | 4 + src/platform/linux/audio/capture_stream.rs | 8 + src/platform/linux/audio/frame.rs | 32 + src/platform/linux/audio/mod.rs | 2 + src/platform/linux/mod.rs | 18 + src/platform/linux/wayland/capture_content.rs | 303 ++++++++++ src/platform/linux/wayland/capture_stream.rs | 566 ++++++++++++++++++ src/platform/linux/wayland/frame.rs | 33 + src/platform/linux/wayland/mod.rs | 3 + src/platform/mod.rs | 4 + 10 files changed, 973 insertions(+) create mode 100644 src/platform/linux/audio/capture_stream.rs create mode 100644 src/platform/linux/audio/frame.rs create mode 100644 src/platform/linux/audio/mod.rs create mode 100644 src/platform/linux/mod.rs create mode 100644 src/platform/linux/wayland/capture_content.rs create mode 100644 src/platform/linux/wayland/capture_stream.rs create mode 100644 src/platform/linux/wayland/frame.rs create mode 100644 src/platform/linux/wayland/mod.rs diff --git a/Cargo.toml b/Cargo.toml index f37e23ea..2f916c65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,6 +86,10 @@ wgpu = { version = "0.20", optional = true, features = ["dx12", "hal"] } d3d12 = "0.20" winapi = { version = "0.3", optional = true } +[target.'cfg(target_os = "linux")'.dependencies] +ashpd = "0.9.1" +pipewire = "0.8.0" + [dev-dependencies] futures = "0.3" tokio = { version = "1.37", features = ["rt", "macros", "rt-multi-thread"] } diff --git a/src/platform/linux/audio/capture_stream.rs b/src/platform/linux/audio/capture_stream.rs new file mode 100644 index 00000000..7dea000c --- /dev/null +++ b/src/platform/linux/audio/capture_stream.rs @@ -0,0 +1,8 @@ +#[derive(Clone, Debug)] +pub struct LinuxAudioCaptureConfig {} + +impl LinuxAudioCaptureConfig { + pub fn new() -> Self { + Self {} + } +} diff --git a/src/platform/linux/audio/frame.rs b/src/platform/linux/audio/frame.rs new file mode 100644 index 00000000..0194a0e0 --- /dev/null +++ b/src/platform/linux/audio/frame.rs @@ -0,0 +1,32 @@ +use crate::prelude::AudioCaptureFrame; + +pub(crate) struct LinuxAudioFrame; + +impl AudioCaptureFrame for LinuxAudioFrame { + fn sample_rate(&self) -> crate::prelude::AudioSampleRate { + todo!() + } + + fn channel_count(&self) -> crate::prelude::AudioChannelCount { + todo!() + } + + fn audio_channel_buffer( + &mut self, + _channel: usize, + ) -> Result, crate::prelude::AudioBufferError> { + todo!() + } + + fn duration(&self) -> std::time::Duration { + todo!() + } + + fn origin_time(&self) -> std::time::Duration { + todo!() + } + + fn frame_id(&self) -> u64 { + todo!() + } +} diff --git a/src/platform/linux/audio/mod.rs b/src/platform/linux/audio/mod.rs new file mode 100644 index 00000000..37ed59c7 --- /dev/null +++ b/src/platform/linux/audio/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod capture_stream; +pub(crate) mod frame; diff --git a/src/platform/linux/mod.rs b/src/platform/linux/mod.rs new file mode 100644 index 00000000..389b5da2 --- /dev/null +++ b/src/platform/linux/mod.rs @@ -0,0 +1,18 @@ +pub(crate) mod audio; +pub(crate) mod wayland; + +pub(crate) use wayland::capture_stream::WaylandCaptureAccessToken as ImplCaptureAccessToken; +pub(crate) use wayland::capture_stream::WaylandCaptureConfig as ImplCaptureConfig; +pub(crate) use wayland::capture_stream::WaylandCaptureStream as ImplCaptureStream; +pub(crate) use wayland::capture_stream::WaylandPixelFormat as ImplPixelFormat; + +pub(crate) use wayland::frame::WaylandVideoFrame as ImplVideoFrame; + +pub(crate) use wayland::capture_content::WaylandCapturableApplication as ImplCapturableApplication; +pub(crate) use wayland::capture_content::WaylandCapturableContent as ImplCapturableContent; +pub(crate) use wayland::capture_content::WaylandCapturableContentFilter as ImplCapturableContentFilter; +pub(crate) use wayland::capture_content::WaylandCapturableDisplay as ImplCapturableDisplay; +pub(crate) use wayland::capture_content::WaylandCapturableWindow as ImplCapturableWindow; + +pub(crate) use audio::capture_stream::LinuxAudioCaptureConfig as ImplAudioCaptureConfig; +pub(crate) use audio::frame::LinuxAudioFrame as ImplAudioFrame; diff --git a/src/platform/linux/wayland/capture_content.rs b/src/platform/linux/wayland/capture_content.rs new file mode 100644 index 00000000..90974c72 --- /dev/null +++ b/src/platform/linux/wayland/capture_content.rs @@ -0,0 +1,303 @@ +use std::rc::Rc; + +use ashpd::{ + desktop::{ + screencast::{CursorMode, Screencast, SourceType}, + Session, + }, + enumflags2::BitFlags, +}; + +use crate::{ + capturable_content::{CapturableContentError, CapturableContentFilter}, + prelude::Rect, +}; + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub struct WaylandCapturableWindow { + pub pw_node_id: u32, + pub position: Option<(i32, i32)>, + pub size: Option<(i32, i32)>, + pub id: Option, + pub mapping_id: Option, + pub virt: bool, + pub cursor_as_metadata: bool, +} + +impl WaylandCapturableWindow { + pub fn from_impl(window: Self) -> Self { + window + } + + pub fn title(&self) -> String { + String::from("n/a") + } + + pub fn rect(&self) -> Rect { + let origin = self.position.unwrap_or((0, 0)); + let size = self.size.unwrap_or((0, 0)); + Rect { + origin: crate::prelude::Point { + x: origin.0 as f64, + y: origin.1 as f64, + }, + size: crate::prelude::Size { + width: size.0 as f64, + height: size.1 as f64, + }, + } + } + + pub fn application(&self) -> WaylandCapturableApplication { + WaylandCapturableApplication(()) + } + + pub fn is_visible(&self) -> bool { + true + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub struct WaylandCapturableDisplay { + pub pw_node_id: u32, + pub position: Option<(i32, i32)>, + pub size: Option<(i32, i32)>, + pub id: Option, + pub mapping_id: Option, + pub cursor_as_metadata: bool, +} + +impl WaylandCapturableDisplay { + pub fn from_impl(window: Self) -> Self { + window + } + + pub fn rect(&self) -> Rect { + let origin = self.position.unwrap_or((0, 0)); + let size = self.size.unwrap_or((0, 0)); + Rect { + origin: crate::prelude::Point { + x: origin.0 as f64, + y: origin.1 as f64, + }, + size: crate::prelude::Size { + width: size.0 as f64, + height: size.1 as f64, + }, + } + } +} + +pub struct WaylandCapturableApplication(()); + +impl WaylandCapturableApplication { + pub fn identifier(&self) -> String { + String::from("n/a") + } + + pub fn name(&self) -> String { + String::from("n/a") + } + + pub fn pid(&self) -> i32 { + -1 + } +} + +pub struct WaylandCapturableContent { + pub windows: Vec, + pub displays: Vec, + _sc: Rc>, + _sc_session: Rc>>, +} + +impl WaylandCapturableContent { + fn source_types_filter(filter: CapturableContentFilter) -> BitFlags { + let mut bitflags = BitFlags::empty(); + if filter.displays { + bitflags |= SourceType::Monitor; + } + if let Some(windows_filter) = filter.windows { + if windows_filter.desktop_windows || windows_filter.onscreen_only { + bitflags |= SourceType::Window | SourceType::Virtual; + } + } + bitflags + } + + pub async fn new(filter: CapturableContentFilter) -> Result { + let screencast = Screencast::new() + .await + .map_err(|e| CapturableContentError::Other(e.to_string()))?; + + let cursor_as_metadata = screencast + .available_cursor_modes() + .await + .map_err(|e| CapturableContentError::Other(e.to_string()))? + .contains(CursorMode::Metadata); + + let session = screencast + .create_session() + .await + .map_err(|e| CapturableContentError::Other(e.to_string()))?; + + screencast + .select_sources( + &session, + // TODO: Show cursor as default when metadata-mode is not available? + if cursor_as_metadata { + CursorMode::Metadata + } else { + CursorMode::Embedded + }, + Self::source_types_filter(filter), + false, + None, + ashpd::desktop::PersistMode::DoNot, + ) + .await + .map_err(|e| CapturableContentError::Other(e.to_string()))? + .response() + .map_err(|e| CapturableContentError::Other(e.to_string()))?; + let streams = screencast + .start(&session, &ashpd::WindowIdentifier::None) + .await + .map_err(|e| CapturableContentError::Other(e.to_string()))? + .response() + .map_err(|e| CapturableContentError::Other(e.to_string()))?; + + let mut sources = Self { + windows: Vec::new(), + displays: Vec::new(), + _sc: Rc::new(screencast), + _sc_session: Rc::new(session), + }; + for stream in streams.streams() { + if let Some(source_type) = stream.source_type() { + match source_type { + SourceType::Window | SourceType::Virtual => { + sources.windows.push(WaylandCapturableWindow { + pw_node_id: stream.pipe_wire_node_id(), + position: stream.position(), + size: stream.size(), + id: if let Some(id) = stream.id() { + Some(id.to_string()) + } else { + None + }, + mapping_id: if let Some(id) = stream.mapping_id() { + Some(id.to_string()) + } else { + None + }, + virt: source_type == SourceType::Virtual, + cursor_as_metadata, + }); + continue; + } + SourceType::Monitor => {} + } + } + // If the stream source_type is `None`, then treat it as a display + sources.displays.push(WaylandCapturableDisplay { + pw_node_id: stream.pipe_wire_node_id(), + position: stream.position(), + size: stream.size(), + id: if let Some(id) = stream.id() { + Some(id.to_string()) + } else { + None + }, + mapping_id: if let Some(id) = stream.mapping_id() { + Some(id.to_string()) + } else { + None + }, + cursor_as_metadata, + }); + } + + Ok(sources) + } +} + +#[derive(Clone, Default)] +pub(crate) struct WaylandCapturableContentFilter(()); + +impl WaylandCapturableContentFilter { + pub(crate) const DEFAULT: Self = Self(()); + pub(crate) const NORMAL_WINDOWS: Self = Self(()); +} + +#[cfg(test)] +mod tests { + use ashpd::{desktop::screencast::SourceType, enumflags2::BitFlags}; + + use crate::{ + platform::platform_impl::{ + wayland::capture_content::WaylandCapturableContent, ImplCapturableContentFilter, + }, + prelude::{CapturableContentFilter, CapturableWindowFilter}, + }; + + #[test] + fn source_type_filter_conversion_displays() { + assert_eq!( + WaylandCapturableContent::source_types_filter(CapturableContentFilter { + windows: None, + displays: true, + impl_capturable_content_filter: ImplCapturableContentFilter::default(), + }), + SourceType::Monitor + ); + } + + #[test] + fn source_type_filter_conversion_windows() { + assert_eq!( + WaylandCapturableContent::source_types_filter(CapturableContentFilter { + windows: Some(CapturableWindowFilter { + desktop_windows: true, + onscreen_only: false + }), + displays: false, + impl_capturable_content_filter: ImplCapturableContentFilter::default(), + }), + SourceType::Window | SourceType::Virtual + ); + assert_eq!( + WaylandCapturableContent::source_types_filter(CapturableContentFilter { + windows: Some(CapturableWindowFilter { + desktop_windows: false, + onscreen_only: true + }), + displays: false, + impl_capturable_content_filter: ImplCapturableContentFilter::default(), + }), + SourceType::Window | SourceType::Virtual + ); + assert_eq!( + WaylandCapturableContent::source_types_filter(CapturableContentFilter { + windows: Some(CapturableWindowFilter { + desktop_windows: true, + onscreen_only: true + }), + displays: false, + impl_capturable_content_filter: ImplCapturableContentFilter::default(), + }), + SourceType::Window | SourceType::Virtual + ); + } + + #[test] + fn source_type_filter_conversion_none() { + assert_eq!( + WaylandCapturableContent::source_types_filter(CapturableContentFilter { + windows: None, + displays: false, + impl_capturable_content_filter: ImplCapturableContentFilter::default(), + }), + BitFlags::empty() + ); + } +} diff --git a/src/platform/linux/wayland/capture_stream.rs b/src/platform/linux/wayland/capture_stream.rs new file mode 100644 index 00000000..7d654228 --- /dev/null +++ b/src/platform/linux/wayland/capture_stream.rs @@ -0,0 +1,566 @@ +use std::{ + cell::RefCell, + ffi::CString, + mem::size_of, + rc::Rc, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::Sender, + Arc, + }, + thread::JoinHandle, + time::Duration, +}; + +use pipewire::{ + context::Context, + main_loop::MainLoop, + spa::{ + self, + param::{ + self, + format::{self, MediaSubtype, MediaType}, + video::{VideoFormat, VideoInfoRaw}, + ParamType, + }, + pod::{self, Pod, Property}, + sys::{ + spa_buffer, spa_meta_bitmap, spa_meta_cursor, spa_meta_header, spa_meta_region, + SPA_META_Bitmap, SPA_META_Cursor, SPA_META_Header, SPA_META_VideoCrop, + SPA_PARAM_META_size, SPA_PARAM_META_type, + }, + utils::{Direction, Fraction, Rectangle, SpaTypes}, + }, + stream::{Stream, StreamFlags, StreamRef, StreamState}, +}; + +use crate::prelude::{ + CaptureConfig, CapturePixelFormat, StreamCreateError, StreamError, StreamEvent, StreamStopError, +}; + +#[derive(Default, Debug, PartialEq)] +struct PwMetas<'a> { + pub header: Option<&'a spa_meta_header>, + pub cursor: Option<&'a spa_meta_cursor>, + // TODO: Get the bitmap: https://docs.pipewire.org/video-play_8c-example.html#_a30 + pub bitmap: Option<&'a spa_meta_bitmap>, +} + +impl<'a> PwMetas<'a> { + pub fn from_raw(raw: &'a *mut spa_buffer) -> Self { + let mut self_ = Self::default(); + + unsafe { + let n_metas = (*(*raw)).n_metas; + if n_metas == 0 { + return self_; + } + + let mut meta_ptr = (*(*raw)).metas; + let metas_end = (*(*raw)).metas.wrapping_add(n_metas as usize); + while meta_ptr != metas_end { + if (*meta_ptr).type_ == SPA_META_Header { + self_.header = Some(&mut *((*meta_ptr).data as *mut spa_meta_header)); + } else if (*meta_ptr).type_ == SPA_META_Cursor { + self_.cursor = Some(&mut *((*meta_ptr).data as *mut spa_meta_cursor)); + } else if (*meta_ptr).type_ == SPA_META_Bitmap { + self_.bitmap = Some(&mut *((*meta_ptr).data as *mut spa_meta_bitmap)) + } + meta_ptr = meta_ptr.wrapping_add(1); + } + } + + self_ + } +} + +#[derive(Default)] +struct PwDatas<'a> { + pub data: &'a [u8], +} + +impl<'a> PwDatas<'a> { + pub fn from_raw(raw: &'a *mut spa_buffer) -> Vec> { + let mut datas = Vec::new(); + + unsafe { + let n_datas = (*(*raw)).n_datas; + if n_datas == 0 { + return datas; + } + + let mut data_ptr = (*(*raw)).datas; + let datas_end = (*(*raw)).datas.wrapping_add(n_datas as usize); + while data_ptr != datas_end { + if !(*data_ptr).data.is_null() { + datas.push(PwDatas { + data: std::slice::from_raw_parts( + (*data_ptr).data as *mut u8, + (*data_ptr).maxsize as usize, + ), + }); + } + data_ptr = data_ptr.wrapping_add(1); + } + } + + datas + } +} + +struct WaylandCapturerUD { + pub format: VideoInfoRaw, + pub format_negotiating: Rc>, + pub show_cursor_as_metadata: bool, +} + +pub struct WaylandCaptureStream { + handle: Option>, + should_run: Arc, +} + +impl WaylandCaptureStream { + pub fn supported_pixel_formats() -> &'static [CapturePixelFormat] { + &[CapturePixelFormat::Bgra8888] + } + + pub fn check_access(_borderless: bool) -> Option { + Some(WaylandCaptureAccessToken(())) + } + + pub async fn request_access(_borderless: bool) -> Option { + Some(WaylandCaptureAccessToken(())) + } + + fn pod_supported_pixel_formats() -> pod::Property { + pipewire::spa::pod::property!( + format::FormatProperties::VideoFormat, + Choice, + Enum, + Id, + VideoFormat::RGBA, // Big-endian + VideoFormat::RGBx, // Big-endian + VideoFormat::BGRx, // Big-endian + VideoFormat::BGRA, // Big-endian + VideoFormat::ABGR, // Big-endian + VideoFormat::ARGB, // Big-endian + VideoFormat::xRGB, // Big-endian + VideoFormat::xBGR // Big-endian + ) + } + + fn pod_supported_resolutions() -> pod::Property { + pod::property!( + format::FormatProperties::VideoSize, + Choice, + Range, + Rectangle, + Rectangle { + width: 512, + height: 512 + }, + Rectangle { + width: 1, + height: 1 + }, + Rectangle { + width: 15360, + height: 8640 + } + ) + } + + fn pod_supported_framerates() -> pod::Property { + pod::property!( + format::FormatProperties::VideoFramerate, + Choice, + Range, + Fraction, + Fraction { num: 30, denom: 1 }, + Fraction { num: 0, denom: 1 }, + Fraction { num: 512, denom: 1 } + ) + } + + fn param_changed(stream: &StreamRef, ud: &mut WaylandCapturerUD, id: u32, param: Option<&Pod>) { + let Some(param) = param else { + return; + }; + + use pipewire::spa::param; + if id != ParamType::Format.as_raw() { + return; + } + + match param::format_utils::parse_format(param) { + Ok((media_type, media_subtype)) => { + if media_type != MediaType::Video || media_subtype != MediaSubtype::Raw { + return; + } + } + Err(e) => { + unsafe { + pipewire::sys::pw_stream_set_error( + stream.as_raw_ptr(), + -1, + CString::new(e.to_string()).unwrap().as_c_str().as_ptr(), + ); + } + return; + } + }; + + let metas_obj = if ud.show_cursor_as_metadata { + pod::object!( + SpaTypes::ObjectParamMeta, + ParamType::Meta, + Property::new( + SPA_PARAM_META_type, + pod::Value::Id(spa::utils::Id(SPA_META_Header)) + ), + Property::new( + SPA_PARAM_META_type, + pod::Value::Id(spa::utils::Id(SPA_META_Cursor)) + ), + Property::new( + SPA_PARAM_META_type, + pod::Value::Id(spa::utils::Id(SPA_META_Bitmap)) + ), + Property::new( + SPA_PARAM_META_size, + pod::Value::Int( + size_of::() as i32 + + size_of::() as i32 + + size_of::() as i32 + ) + ), + ) + } else { + pod::object!( + SpaTypes::ObjectParamMeta, + ParamType::Meta, + Property::new( + SPA_PARAM_META_type, + pod::Value::Id(spa::utils::Id(SPA_META_Header)) + ), + Property::new( + SPA_PARAM_META_size, + pod::Value::Int(size_of::() as i32) + ), + ) + }; + + let metas_values: Vec = pod::serialize::PodSerializer::serialize( + std::io::Cursor::new(Vec::new()), + &pod::Value::Object(metas_obj), + ) + .unwrap() + .0 + .into_inner(); + + let mut params = [pod::Pod::from_bytes(&metas_values).unwrap()]; + + if let Err(e) = stream.update_params(&mut params) { + unsafe { + pipewire::sys::pw_stream_set_error( + stream.as_raw_ptr(), + -1, + CString::new(e.to_string()).unwrap().as_c_str().as_ptr(), + ); + } + return; + } + + ud.format.parse(param).unwrap(); + println!( + "Got pixel format: {} ({:?})", + ud.format.format().as_raw(), + ud.format.format() + ); + + ud.format_negotiating.replace(false); + } + + fn process(stream: &StreamRef, _ud: &mut WaylandCapturerUD) { + let raw_buffer = unsafe { stream.dequeue_raw_buffer() }; + if raw_buffer.is_null() { + unsafe { stream.queue_raw_buffer(raw_buffer) }; + return; + } + + let buffer = unsafe { (*raw_buffer).buffer }; + if buffer.is_null() { + unsafe { stream.queue_raw_buffer(raw_buffer) }; + return; + } + + let metas = PwMetas::from_raw(&buffer); + let datas = PwDatas::from_raw(&buffer); + println!("n_datas: {}", datas.len()); + + unsafe { stream.queue_raw_buffer(raw_buffer) }; + } + + fn capture_main( + capture_config: CaptureConfig, + callback: Box) + Send + 'static>, + should_run: Arc, + init_tx: &Sender>, + ) -> Result<(), StreamCreateError> { + pipewire::init(); + let mainloop = MainLoop::new(None).map_err(|e| StreamCreateError::Other(e.to_string()))?; + let context = + Context::new(&mainloop).map_err(|e| StreamCreateError::Other(e.to_string()))?; + let core = context + .connect(None) + .map_err(|e| StreamCreateError::Other(e.to_string()))?; + + use pipewire::keys; + let stream = Stream::new( + &core, + "crabgrab", + pipewire::properties::properties! { + *keys::MEDIA_TYPE => "Video", + *keys::MEDIA_CATEGORY => "Capture", + *keys::MEDIA_ROLE => "Screen", + }, + ) + .map_err(|e| StreamCreateError::Other(e.to_string()))?; + + let format_negotiating = Rc::new(RefCell::new(true)); + let user_data = WaylandCapturerUD { + format: Default::default(), + format_negotiating: format_negotiating.clone(), + show_cursor_as_metadata: capture_config.show_cursor + && match &capture_config.target { + crate::prelude::Capturable::Window(w) => { + w.impl_capturable_window.cursor_as_metadata + } + crate::prelude::Capturable::Display(d) => { + d.impl_capturable_display.cursor_as_metadata + } + }, + }; + + let _listener = stream + .add_local_listener_with_user_data(user_data) + .param_changed(Self::param_changed) + .process(Self::process) + .register() + .map_err(|e| StreamCreateError::Other(e.to_string()))?; + + let stream_param_obj = pod::object!( + spa::utils::SpaTypes::ObjectParamFormat, + param::ParamType::EnumFormat, + pod::property!( + format::FormatProperties::MediaType, + Id, + format::MediaType::Video + ), + pod::property!( + format::FormatProperties::MediaSubtype, + Id, + format::MediaSubtype::Raw + ), + Self::pod_supported_pixel_formats(), + Self::pod_supported_resolutions(), + Self::pod_supported_framerates(), + ); + + let stream_param_obj_values: Vec = pod::serialize::PodSerializer::serialize( + std::io::Cursor::new(Vec::new()), + &pod::Value::Object(stream_param_obj), + ) + .map_err(|e| StreamCreateError::Other(e.to_string()))? + .0 + .into_inner(); + + let mut params = [pod::Pod::from_bytes(&stream_param_obj_values).unwrap()]; + + stream + .connect( + Direction::Input, + Some(match capture_config.target { + crate::prelude::Capturable::Window(w) => w.impl_capturable_window.pw_node_id, + crate::prelude::Capturable::Display(d) => d.impl_capturable_display.pw_node_id, + }), + StreamFlags::AUTOCONNECT | StreamFlags::MAP_BUFFERS, + &mut params, + ) + .map_err(|e| StreamCreateError::Other(e.to_string()))?; + + let loop_ = mainloop.loop_(); + + // Iterate the stream and check for errors while negotiating pixel format + // BUG: Does not exit when supported format is not found + while *(*format_negotiating).borrow() { + loop_.iterate(Duration::from_millis(100)); + match stream.state() { + StreamState::Error(_) => { + return Err(StreamCreateError::UnsupportedPixelFormat); + } + _ => {} + } + } + + init_tx.send(Ok(())).unwrap(); + + while should_run.load(Ordering::Acquire) { + loop_.iterate(Duration::from_millis(100)); + } + + let _ = stream.disconnect(); + + Ok(()) + } + + pub fn new( + _token: WaylandCaptureAccessToken, + capture_config: CaptureConfig, + callback: Box) + Send + 'static>, + ) -> Result { + let should_run = Arc::new(AtomicBool::new(true)); + let should_run_clone = Arc::clone(&should_run); + let (init_tx, init_rx) = std::sync::mpsc::channel::>(); + let handle = std::thread::spawn(move || { + if let Err(e) = Self::capture_main(capture_config, callback, should_run_clone, &init_tx) + { + init_tx.send(Err(e)).unwrap(); + } + }); + + init_rx.recv().unwrap()?; + + Ok(Self { + handle: Some(handle), + should_run, + }) + } + + pub(crate) fn stop(&mut self) -> Result<(), StreamStopError> { + if self.should_run.load(Ordering::Acquire) { + self.should_run + .store(false, std::sync::atomic::Ordering::SeqCst); + } + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + Ok(()) + } +} + +impl Drop for WaylandCaptureStream { + fn drop(&mut self) { + let _ = self.stop(); + } +} + +#[derive(Clone, Debug)] +pub struct WaylandCaptureConfig {} + +impl WaylandCaptureConfig { + pub fn new() -> Self { + Self {} + } +} + +pub struct WaylandPixelFormat {} + +#[derive(Clone, Copy, Debug)] +pub struct WaylandCaptureAccessToken(()); + +impl WaylandCaptureAccessToken { + pub(crate) fn allows_borderless(&self) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use std::ffi::c_void; + + use pipewire::spa::sys::spa_meta; + + use super::*; + + #[test] + fn buffer_metas_extraction_header() { + let mut meta_header_data = spa_meta_header { + flags: 1, + offset: 2, + pts: 3, + dts_offset: 4, + seq: 5, + }; + let mut metas = [spa_meta { + type_: SPA_META_Header, + size: std::mem::size_of_val(&meta_header_data) as u32, + data: std::ptr::addr_of_mut!(meta_header_data) as *mut c_void, + }]; + let mut buffer = spa_buffer { + n_metas: metas.len() as u32, + n_datas: 0, + metas: std::ptr::addr_of_mut!(metas) as *mut spa_meta, + datas: std::ptr::null_mut(), + }; + let buffer_addr = std::ptr::addr_of_mut!(buffer); + let extracted_metas = PwMetas::from_raw(&buffer_addr); + assert_eq!( + extracted_metas, + PwMetas { header: Some(&meta_header_data), cursor: None, bitmap: None } + ); + } + + #[test] + fn buffer_datas_extraction_header_cursor_bitmap() { + let mut meta_header_data = spa_meta_header { + flags: 1, + offset: 2, + pts: 3, + dts_offset: 4, + seq: 5, + }; + let mut meta_cursor_data = spa_meta_cursor { + id: 0, + flags: 123, + position: spa::sys::spa_point { x: 10, y: 12 }, + hotspot: spa::sys::spa_point { x: 20, y: 22 }, + bitmap_offset: 321, + }; + let mut meta_bitmap_data = spa_meta_bitmap { + format: 0, + size: spa::sys::spa_rectangle { width: 0, height: 0 }, + stride: 0, + offset: 5, + }; + let mut metas = [ + spa_meta { + type_: SPA_META_Header, + size: std::mem::size_of_val(&meta_header_data) as u32, + data: std::ptr::addr_of_mut!(meta_header_data) as *mut c_void, + }, + spa_meta { + type_: SPA_META_Cursor, + size: std::mem::size_of_val(&meta_cursor_data) as u32, + data: std::ptr::addr_of_mut!(meta_cursor_data) as *mut c_void, + }, + spa_meta { + type_: SPA_META_Bitmap, + size: std::mem::size_of_val(&meta_bitmap_data) as u32, + data: std::ptr::addr_of_mut!(meta_bitmap_data) as *mut c_void, + } + ]; + let mut buffer = spa_buffer { + n_metas: metas.len() as u32, + n_datas: 0, + metas: std::ptr::addr_of_mut!(metas) as *mut spa_meta, + datas: std::ptr::null_mut(), + }; + let buffer_addr = std::ptr::addr_of_mut!(buffer); + let extracted_metas = PwMetas::from_raw(&buffer_addr); + assert_eq!( + extracted_metas, + PwMetas { header: Some(&meta_header_data), cursor: Some(&meta_cursor_data), bitmap: Some(&meta_bitmap_data) } + ); + } +} diff --git a/src/platform/linux/wayland/frame.rs b/src/platform/linux/wayland/frame.rs new file mode 100644 index 00000000..e87c0d33 --- /dev/null +++ b/src/platform/linux/wayland/frame.rs @@ -0,0 +1,33 @@ +use crate::prelude::VideoCaptureFrame; + +pub(crate) struct WaylandVideoFrame {} + +impl VideoCaptureFrame for WaylandVideoFrame { + fn size(&self) -> crate::prelude::Size { + todo!() + } + + fn dpi(&self) -> f64 { + todo!() + } + + fn duration(&self) -> std::time::Duration { + todo!() + } + + fn origin_time(&self) -> std::time::Duration { + todo!() + } + + fn capture_time(&self) -> std::time::Instant { + todo!() + } + + fn frame_id(&self) -> u64 { + todo!() + } + + fn content_rect(&self) -> crate::prelude::Rect { + todo!() + } +} diff --git a/src/platform/linux/wayland/mod.rs b/src/platform/linux/wayland/mod.rs new file mode 100644 index 00000000..e20f3b6f --- /dev/null +++ b/src/platform/linux/wayland/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod capture_content; +pub(crate) mod capture_stream; +pub(crate) mod frame; diff --git a/src/platform/mod.rs b/src/platform/mod.rs index b2f0d4d3..6cf7f35c 100644 --- a/src/platform/mod.rs +++ b/src/platform/mod.rs @@ -12,4 +12,8 @@ pub mod windows; #[cfg(target_os = "windows")] pub(crate) use windows as platform_impl; +#[cfg(target_os = "linux")] +pub mod linux; +#[cfg(target_os = "linux")] +pub(crate) use linux as platform_impl; From b848e288fb81029f9fdd5b810f32ec44989c033f Mon Sep 17 00:00:00 2001 From: Marcus Date: Tue, 23 Jul 2024 19:34:09 +0200 Subject: [PATCH 02/10] linux-wayland: fix test name typo --- src/platform/linux/wayland/capture_stream.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/platform/linux/wayland/capture_stream.rs b/src/platform/linux/wayland/capture_stream.rs index 7d654228..56b95a60 100644 --- a/src/platform/linux/wayland/capture_stream.rs +++ b/src/platform/linux/wayland/capture_stream.rs @@ -392,7 +392,7 @@ impl WaylandCaptureStream { let loop_ = mainloop.loop_(); // Iterate the stream and check for errors while negotiating pixel format - // BUG: Does not exit when supported format is not found + // BUG: Does not exit when supported format is not found. Find out if it timed out while *(*format_negotiating).borrow() { loop_.iterate(Duration::from_millis(100)); match stream.state() { @@ -512,7 +512,7 @@ mod tests { } #[test] - fn buffer_datas_extraction_header_cursor_bitmap() { + fn buffer_metas_extraction_header_cursor_bitmap() { let mut meta_header_data = spa_meta_header { flags: 1, offset: 2, From 8f05dae5de0a1d812293de79d561fd8ced741de1 Mon Sep 17 00:00:00 2001 From: Marcus Date: Wed, 24 Jul 2024 21:31:11 +0200 Subject: [PATCH 03/10] linux-wayland: rename implementation directory --- src/platform/linux/audio/capture_stream.rs | 8 --- src/platform/linux/audio/frame.rs | 32 ---------- src/platform/linux/audio/mod.rs | 2 - src/platform/linux/mod.rs | 18 ------ src/platform/linux/wayland/mod.rs | 3 - .../capture_content.rs | 0 .../capture_stream.rs | 0 .../{linux/wayland => linux_wayland}/frame.rs | 0 src/platform/linux_wayland/mod.rs | 59 +++++++++++++++++++ 9 files changed, 59 insertions(+), 63 deletions(-) delete mode 100644 src/platform/linux/audio/capture_stream.rs delete mode 100644 src/platform/linux/audio/frame.rs delete mode 100644 src/platform/linux/audio/mod.rs delete mode 100644 src/platform/linux/mod.rs delete mode 100644 src/platform/linux/wayland/mod.rs rename src/platform/{linux/wayland => linux_wayland}/capture_content.rs (100%) rename src/platform/{linux/wayland => linux_wayland}/capture_stream.rs (100%) rename src/platform/{linux/wayland => linux_wayland}/frame.rs (100%) create mode 100644 src/platform/linux_wayland/mod.rs diff --git a/src/platform/linux/audio/capture_stream.rs b/src/platform/linux/audio/capture_stream.rs deleted file mode 100644 index 7dea000c..00000000 --- a/src/platform/linux/audio/capture_stream.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[derive(Clone, Debug)] -pub struct LinuxAudioCaptureConfig {} - -impl LinuxAudioCaptureConfig { - pub fn new() -> Self { - Self {} - } -} diff --git a/src/platform/linux/audio/frame.rs b/src/platform/linux/audio/frame.rs deleted file mode 100644 index 0194a0e0..00000000 --- a/src/platform/linux/audio/frame.rs +++ /dev/null @@ -1,32 +0,0 @@ -use crate::prelude::AudioCaptureFrame; - -pub(crate) struct LinuxAudioFrame; - -impl AudioCaptureFrame for LinuxAudioFrame { - fn sample_rate(&self) -> crate::prelude::AudioSampleRate { - todo!() - } - - fn channel_count(&self) -> crate::prelude::AudioChannelCount { - todo!() - } - - fn audio_channel_buffer( - &mut self, - _channel: usize, - ) -> Result, crate::prelude::AudioBufferError> { - todo!() - } - - fn duration(&self) -> std::time::Duration { - todo!() - } - - fn origin_time(&self) -> std::time::Duration { - todo!() - } - - fn frame_id(&self) -> u64 { - todo!() - } -} diff --git a/src/platform/linux/audio/mod.rs b/src/platform/linux/audio/mod.rs deleted file mode 100644 index 37ed59c7..00000000 --- a/src/platform/linux/audio/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod capture_stream; -pub(crate) mod frame; diff --git a/src/platform/linux/mod.rs b/src/platform/linux/mod.rs deleted file mode 100644 index 389b5da2..00000000 --- a/src/platform/linux/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -pub(crate) mod audio; -pub(crate) mod wayland; - -pub(crate) use wayland::capture_stream::WaylandCaptureAccessToken as ImplCaptureAccessToken; -pub(crate) use wayland::capture_stream::WaylandCaptureConfig as ImplCaptureConfig; -pub(crate) use wayland::capture_stream::WaylandCaptureStream as ImplCaptureStream; -pub(crate) use wayland::capture_stream::WaylandPixelFormat as ImplPixelFormat; - -pub(crate) use wayland::frame::WaylandVideoFrame as ImplVideoFrame; - -pub(crate) use wayland::capture_content::WaylandCapturableApplication as ImplCapturableApplication; -pub(crate) use wayland::capture_content::WaylandCapturableContent as ImplCapturableContent; -pub(crate) use wayland::capture_content::WaylandCapturableContentFilter as ImplCapturableContentFilter; -pub(crate) use wayland::capture_content::WaylandCapturableDisplay as ImplCapturableDisplay; -pub(crate) use wayland::capture_content::WaylandCapturableWindow as ImplCapturableWindow; - -pub(crate) use audio::capture_stream::LinuxAudioCaptureConfig as ImplAudioCaptureConfig; -pub(crate) use audio::frame::LinuxAudioFrame as ImplAudioFrame; diff --git a/src/platform/linux/wayland/mod.rs b/src/platform/linux/wayland/mod.rs deleted file mode 100644 index e20f3b6f..00000000 --- a/src/platform/linux/wayland/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) mod capture_content; -pub(crate) mod capture_stream; -pub(crate) mod frame; diff --git a/src/platform/linux/wayland/capture_content.rs b/src/platform/linux_wayland/capture_content.rs similarity index 100% rename from src/platform/linux/wayland/capture_content.rs rename to src/platform/linux_wayland/capture_content.rs diff --git a/src/platform/linux/wayland/capture_stream.rs b/src/platform/linux_wayland/capture_stream.rs similarity index 100% rename from src/platform/linux/wayland/capture_stream.rs rename to src/platform/linux_wayland/capture_stream.rs diff --git a/src/platform/linux/wayland/frame.rs b/src/platform/linux_wayland/frame.rs similarity index 100% rename from src/platform/linux/wayland/frame.rs rename to src/platform/linux_wayland/frame.rs diff --git a/src/platform/linux_wayland/mod.rs b/src/platform/linux_wayland/mod.rs new file mode 100644 index 00000000..c3c42f8f --- /dev/null +++ b/src/platform/linux_wayland/mod.rs @@ -0,0 +1,59 @@ +mod capture_stream; +mod frame; +mod capture_content; + +pub(crate) use capture_stream::WaylandCaptureAccessToken as ImplCaptureAccessToken; +pub(crate) use capture_stream::WaylandCaptureConfig as ImplCaptureConfig; +pub(crate) use capture_stream::WaylandCaptureStream as ImplCaptureStream; +pub(crate) use capture_stream::WaylandPixelFormat as ImplPixelFormat; + +pub(crate) use frame::WaylandVideoFrame as ImplVideoFrame; + +pub(crate) use capture_content::WaylandCapturableApplication as ImplCapturableApplication; +pub(crate) use capture_content::WaylandCapturableContent as ImplCapturableContent; +pub(crate) use capture_content::WaylandCapturableContentFilter as ImplCapturableContentFilter; +pub(crate) use capture_content::WaylandCapturableDisplay as ImplCapturableDisplay; +pub(crate) use capture_content::WaylandCapturableWindow as ImplCapturableWindow; + + +#[derive(Clone, Debug)] +pub(crate) struct ImplAudioCaptureConfig {} + +impl ImplAudioCaptureConfig { + pub fn new() -> Self { + Self {} + } +} + +use crate::prelude::AudioCaptureFrame; + +pub(crate) struct ImplAudioFrame; + +impl AudioCaptureFrame for ImplAudioFrame { + fn sample_rate(&self) -> crate::prelude::AudioSampleRate { + todo!() + } + + fn channel_count(&self) -> crate::prelude::AudioChannelCount { + todo!() + } + + fn audio_channel_buffer( + &mut self, + _channel: usize, + ) -> Result, crate::prelude::AudioBufferError> { + todo!() + } + + fn duration(&self) -> std::time::Duration { + todo!() + } + + fn origin_time(&self) -> std::time::Duration { + todo!() + } + + fn frame_id(&self) -> u64 { + todo!() + } +} From 969805920227738b2d45c57c1f52df841cdf5f14 Mon Sep 17 00:00:00 2001 From: Marcus Date: Thu, 25 Jul 2024 21:09:33 +0200 Subject: [PATCH 04/10] linux-wayland: send frame to callback --- src/platform/linux_wayland/capture_stream.rs | 64 ++++++++++++++++---- src/platform/linux_wayland/frame.rs | 21 +++++-- src/platform/linux_wayland/mod.rs | 1 + 3 files changed, 69 insertions(+), 17 deletions(-) diff --git a/src/platform/linux_wayland/capture_stream.rs b/src/platform/linux_wayland/capture_stream.rs index 56b95a60..1da1be8d 100644 --- a/src/platform/linux_wayland/capture_stream.rs +++ b/src/platform/linux_wayland/capture_stream.rs @@ -25,8 +25,8 @@ use pipewire::{ }, pod::{self, Pod, Property}, sys::{ - spa_buffer, spa_meta_bitmap, spa_meta_cursor, spa_meta_header, spa_meta_region, - SPA_META_Bitmap, SPA_META_Cursor, SPA_META_Header, SPA_META_VideoCrop, + spa_buffer, spa_meta_bitmap, spa_meta_cursor, spa_meta_header, + SPA_META_Bitmap, SPA_META_Cursor, SPA_META_Header, SPA_PARAM_META_size, SPA_PARAM_META_type, }, utils::{Direction, Fraction, Rectangle, SpaTypes}, @@ -34,10 +34,16 @@ use pipewire::{ stream::{Stream, StreamFlags, StreamRef, StreamState}, }; -use crate::prelude::{ - CaptureConfig, CapturePixelFormat, StreamCreateError, StreamError, StreamEvent, StreamStopError, +use crate::{ + frame::VideoFrame, + prelude::{ + CaptureConfig, CapturePixelFormat, StreamCreateError, StreamError, StreamEvent, + StreamStopError, + }, }; +use super::frame::WaylandVideoFrame; + #[derive(Default, Debug, PartialEq)] struct PwMetas<'a> { pub header: Option<&'a spa_meta_header>, @@ -112,6 +118,8 @@ struct WaylandCapturerUD { pub format: VideoInfoRaw, pub format_negotiating: Rc>, pub show_cursor_as_metadata: bool, + pub start_time: i64, + pub callback: Box) + Send + 'static>, } pub struct WaylandCaptureStream { @@ -272,7 +280,7 @@ impl WaylandCaptureStream { } ud.format.parse(param).unwrap(); - println!( + println!( // DEBUGGING "Got pixel format: {} ({:?})", ud.format.format().as_raw(), ud.format.format() @@ -281,7 +289,7 @@ impl WaylandCaptureStream { ud.format_negotiating.replace(false); } - fn process(stream: &StreamRef, _ud: &mut WaylandCapturerUD) { + fn process(stream: &StreamRef, ud: &mut WaylandCapturerUD) { let raw_buffer = unsafe { stream.dequeue_raw_buffer() }; if raw_buffer.is_null() { unsafe { stream.queue_raw_buffer(raw_buffer) }; @@ -296,7 +304,27 @@ impl WaylandCaptureStream { let metas = PwMetas::from_raw(&buffer); let datas = PwDatas::from_raw(&buffer); - println!("n_datas: {}", datas.len()); + if let (Some(header), Some(data)) = (metas.header, datas.iter().next()) { + if ud.start_time == 0 { + ud.start_time = header.pts; + } + + let frame = WaylandVideoFrame { + size: crate::prelude::Size { + width: ud.format.size().width as f64, + height: ud.format.size().height as f64, + }, + id: header.seq, + captured: std::time::Instant::now(), + pts: std::time::Duration::from_nanos((header.pts - ud.start_time) as u64), + format: ud.format, + data: data.data.to_vec(), + }; + + (*ud.callback)(Ok(StreamEvent::Video(VideoFrame { + impl_video_frame: frame, + }))); + } unsafe { stream.queue_raw_buffer(raw_buffer) }; } @@ -340,6 +368,8 @@ impl WaylandCaptureStream { d.impl_capturable_display.cursor_as_metadata } }, + start_time: 0, + callback, }; let _listener = stream @@ -464,6 +494,7 @@ impl WaylandCaptureConfig { } } +#[allow(dead_code)] pub struct WaylandPixelFormat {} #[derive(Clone, Copy, Debug)] @@ -507,7 +538,11 @@ mod tests { let extracted_metas = PwMetas::from_raw(&buffer_addr); assert_eq!( extracted_metas, - PwMetas { header: Some(&meta_header_data), cursor: None, bitmap: None } + PwMetas { + header: Some(&meta_header_data), + cursor: None, + bitmap: None + } ); } @@ -529,7 +564,10 @@ mod tests { }; let mut meta_bitmap_data = spa_meta_bitmap { format: 0, - size: spa::sys::spa_rectangle { width: 0, height: 0 }, + size: spa::sys::spa_rectangle { + width: 0, + height: 0, + }, stride: 0, offset: 5, }; @@ -548,7 +586,7 @@ mod tests { type_: SPA_META_Bitmap, size: std::mem::size_of_val(&meta_bitmap_data) as u32, data: std::ptr::addr_of_mut!(meta_bitmap_data) as *mut c_void, - } + }, ]; let mut buffer = spa_buffer { n_metas: metas.len() as u32, @@ -560,7 +598,11 @@ mod tests { let extracted_metas = PwMetas::from_raw(&buffer_addr); assert_eq!( extracted_metas, - PwMetas { header: Some(&meta_header_data), cursor: Some(&meta_cursor_data), bitmap: Some(&meta_bitmap_data) } + PwMetas { + header: Some(&meta_header_data), + cursor: Some(&meta_cursor_data), + bitmap: Some(&meta_bitmap_data) + } ); } } diff --git a/src/platform/linux_wayland/frame.rs b/src/platform/linux_wayland/frame.rs index e87c0d33..cfdf7901 100644 --- a/src/platform/linux_wayland/frame.rs +++ b/src/platform/linux_wayland/frame.rs @@ -1,10 +1,19 @@ +use pipewire::spa::param::video::VideoInfoRaw; + use crate::prelude::VideoCaptureFrame; -pub(crate) struct WaylandVideoFrame {} +pub(crate) struct WaylandVideoFrame { + pub(crate) size: crate::prelude::Size, + pub(crate) id: u64, + pub(crate) captured: std::time::Instant, + pub(crate) pts: std::time::Duration, + pub(crate) format: VideoInfoRaw, + pub(crate) data: Vec, // TODO: Optimize +} impl VideoCaptureFrame for WaylandVideoFrame { fn size(&self) -> crate::prelude::Size { - todo!() + self.size } fn dpi(&self) -> f64 { @@ -12,19 +21,19 @@ impl VideoCaptureFrame for WaylandVideoFrame { } fn duration(&self) -> std::time::Duration { - todo!() + std::time::Duration::from_secs(0) } fn origin_time(&self) -> std::time::Duration { - todo!() + self.pts } fn capture_time(&self) -> std::time::Instant { - todo!() + self.captured } fn frame_id(&self) -> u64 { - todo!() + self.id } fn content_rect(&self) -> crate::prelude::Rect { diff --git a/src/platform/linux_wayland/mod.rs b/src/platform/linux_wayland/mod.rs index c3c42f8f..0ed0b3f2 100644 --- a/src/platform/linux_wayland/mod.rs +++ b/src/platform/linux_wayland/mod.rs @@ -5,6 +5,7 @@ mod capture_content; pub(crate) use capture_stream::WaylandCaptureAccessToken as ImplCaptureAccessToken; pub(crate) use capture_stream::WaylandCaptureConfig as ImplCaptureConfig; pub(crate) use capture_stream::WaylandCaptureStream as ImplCaptureStream; +#[allow(unused_imports)] pub(crate) use capture_stream::WaylandPixelFormat as ImplPixelFormat; pub(crate) use frame::WaylandVideoFrame as ImplVideoFrame; From ef062a195e3a1ab5a3d1ca7596c86d2531a82956 Mon Sep 17 00:00:00 2001 From: Marcus Date: Thu, 25 Jul 2024 22:21:21 +0200 Subject: [PATCH 05/10] linux-wayland: handle disconnects and errors in stream Callback for state_changed event so that stream errors and unconnected events can be handled and notify consumer(s) --- src/platform/linux_wayland/capture_stream.rs | 30 ++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/platform/linux_wayland/capture_stream.rs b/src/platform/linux_wayland/capture_stream.rs index 1da1be8d..585cd6d2 100644 --- a/src/platform/linux_wayland/capture_stream.rs +++ b/src/platform/linux_wayland/capture_stream.rs @@ -120,6 +120,7 @@ struct WaylandCapturerUD { pub show_cursor_as_metadata: bool, pub start_time: i64, pub callback: Box) + Send + 'static>, + pub should_run: Arc, } pub struct WaylandCaptureStream { @@ -190,6 +191,20 @@ impl WaylandCaptureStream { ) } + fn state_changed(_stream: &StreamRef, ud: &mut WaylandCapturerUD, _old: StreamState, new: StreamState) { + match new { + StreamState::Error(e) => { + (*ud.callback)(Err(StreamError::Other(e))); + ud.should_run.store(false, Ordering::SeqCst); + } + StreamState::Unconnected => { + (*ud.callback)(Ok(StreamEvent::End)); + ud.should_run.store(false, Ordering::SeqCst); + } + _ => {} + } + } + fn param_changed(stream: &StreamRef, ud: &mut WaylandCapturerUD, id: u32, param: Option<&Pod>) { let Some(param) = param else { return; @@ -370,10 +385,12 @@ impl WaylandCaptureStream { }, start_time: 0, callback, + should_run: Arc::clone(&should_run), }; let _listener = stream .add_local_listener_with_user_data(user_data) + .state_changed(Self::state_changed) .param_changed(Self::param_changed) .process(Self::process) .register() @@ -422,15 +439,12 @@ impl WaylandCaptureStream { let loop_ = mainloop.loop_(); // Iterate the stream and check for errors while negotiating pixel format - // BUG: Does not exit when supported format is not found. Find out if it timed out while *(*format_negotiating).borrow() { loop_.iterate(Duration::from_millis(100)); - match stream.state() { - StreamState::Error(_) => { - return Err(StreamCreateError::UnsupportedPixelFormat); - } - _ => {} - } + } + + if !should_run.load(Ordering::Acquire) { + return Err(StreamCreateError::UnsupportedPixelFormat); } init_tx.send(Ok(())).unwrap(); @@ -470,7 +484,7 @@ impl WaylandCaptureStream { pub(crate) fn stop(&mut self) -> Result<(), StreamStopError> { if self.should_run.load(Ordering::Acquire) { self.should_run - .store(false, std::sync::atomic::Ordering::SeqCst); + .store(false, Ordering::SeqCst); } if let Some(handle) = self.handle.take() { let _ = handle.join(); From 0bb9858b9b0a73ef1d4c25357ea39092b7c31d5e Mon Sep 17 00:00:00 2001 From: Marcus Date: Fri, 26 Jul 2024 16:57:47 +0200 Subject: [PATCH 06/10] linux-wayland: handle unsupported content filter + treat virtual sources as display --- src/platform/linux_wayland/capture_content.rs | 37 ++++++++++++++----- src/platform/mod.rs | 6 +-- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/platform/linux_wayland/capture_content.rs b/src/platform/linux_wayland/capture_content.rs index 90974c72..5d34d24f 100644 --- a/src/platform/linux_wayland/capture_content.rs +++ b/src/platform/linux_wayland/capture_content.rs @@ -115,11 +115,11 @@ impl WaylandCapturableContent { fn source_types_filter(filter: CapturableContentFilter) -> BitFlags { let mut bitflags = BitFlags::empty(); if filter.displays { - bitflags |= SourceType::Monitor; + bitflags |= SourceType::Monitor | SourceType::Virtual; } if let Some(windows_filter) = filter.windows { if windows_filter.desktop_windows || windows_filter.onscreen_only { - bitflags |= SourceType::Window | SourceType::Virtual; + bitflags |= SourceType::Window; } } bitflags @@ -136,6 +136,10 @@ impl WaylandCapturableContent { .map_err(|e| CapturableContentError::Other(e.to_string()))? .contains(CursorMode::Metadata); + let source_types = Self::source_types_filter(filter) + // Some portal implementations freak out when we include supported an not supported source types + & screencast.available_source_types().await.map_err(|e| CapturableContentError::Other(e.to_string()))?; + let session = screencast .create_session() .await @@ -144,13 +148,13 @@ impl WaylandCapturableContent { screencast .select_sources( &session, - // TODO: Show cursor as default when metadata-mode is not available? + // INVESTIGATE: Show cursor as default when metadata-mode is not available? if cursor_as_metadata { CursorMode::Metadata } else { CursorMode::Embedded }, - Self::source_types_filter(filter), + source_types, false, None, ashpd::desktop::PersistMode::DoNot, @@ -235,7 +239,7 @@ mod tests { use crate::{ platform::platform_impl::{ - wayland::capture_content::WaylandCapturableContent, ImplCapturableContentFilter, + capture_content::WaylandCapturableContent, ImplCapturableContentFilter, }, prelude::{CapturableContentFilter, CapturableWindowFilter}, }; @@ -248,7 +252,7 @@ mod tests { displays: true, impl_capturable_content_filter: ImplCapturableContentFilter::default(), }), - SourceType::Monitor + SourceType::Monitor | SourceType::Virtual ); } @@ -263,7 +267,7 @@ mod tests { displays: false, impl_capturable_content_filter: ImplCapturableContentFilter::default(), }), - SourceType::Window | SourceType::Virtual + SourceType::Window ); assert_eq!( WaylandCapturableContent::source_types_filter(CapturableContentFilter { @@ -274,7 +278,7 @@ mod tests { displays: false, impl_capturable_content_filter: ImplCapturableContentFilter::default(), }), - SourceType::Window | SourceType::Virtual + SourceType::Window ); assert_eq!( WaylandCapturableContent::source_types_filter(CapturableContentFilter { @@ -285,7 +289,7 @@ mod tests { displays: false, impl_capturable_content_filter: ImplCapturableContentFilter::default(), }), - SourceType::Window | SourceType::Virtual + SourceType::Window ); } @@ -300,4 +304,19 @@ mod tests { BitFlags::empty() ); } + + #[test] + fn source_type_filter_conversion_all() { + assert_eq!( + WaylandCapturableContent::source_types_filter(CapturableContentFilter { + windows: Some(CapturableWindowFilter { + desktop_windows: true, + onscreen_only: true + }), + displays: true, + impl_capturable_content_filter: ImplCapturableContentFilter::default(), + }), + SourceType::Monitor | SourceType::Virtual | SourceType::Window + ); + } } diff --git a/src/platform/mod.rs b/src/platform/mod.rs index 6cf7f35c..17c54a1a 100644 --- a/src/platform/mod.rs +++ b/src/platform/mod.rs @@ -10,10 +10,10 @@ pub(crate) use macos as platform_impl; pub mod windows; #[cfg(target_os = "windows")] -pub(crate) use windows as platform_impl; +pub(crate) use windows as platform_impl; #[cfg(target_os = "linux")] -pub mod linux; +pub mod linux_wayland; #[cfg(target_os = "linux")] -pub(crate) use linux as platform_impl; +pub(crate) use linux_wayland as platform_impl; From 43d54beb016cc2488621042818b0992f0c66687c Mon Sep 17 00:00:00 2001 From: Marcus Date: Tue, 30 Jul 2024 18:18:51 +0200 Subject: [PATCH 07/10] linux-wayland: composit metadata cursor onto frame --- src/platform/linux_wayland/capture_content.rs | 44 +-- src/platform/linux_wayland/capture_stream.rs | 321 ++++++++++-------- src/platform/linux_wayland/mod.rs | 3 +- 3 files changed, 197 insertions(+), 171 deletions(-) diff --git a/src/platform/linux_wayland/capture_content.rs b/src/platform/linux_wayland/capture_content.rs index 5d34d24f..32436722 100644 --- a/src/platform/linux_wayland/capture_content.rs +++ b/src/platform/linux_wayland/capture_content.rs @@ -140,20 +140,28 @@ impl WaylandCapturableContent { // Some portal implementations freak out when we include supported an not supported source types & screencast.available_source_types().await.map_err(|e| CapturableContentError::Other(e.to_string()))?; + if source_types.is_empty() { + return Err(CapturableContentError::Other( + "Unsupported content filter".to_string(), + )); + } + let session = screencast .create_session() .await .map_err(|e| CapturableContentError::Other(e.to_string()))?; + // INVESTIGATE: Show cursor as default when metadata-mode is not available? + let cursor_mode = if cursor_as_metadata { + CursorMode::Metadata + } else { + CursorMode::Embedded + }; + screencast .select_sources( &session, - // INVESTIGATE: Show cursor as default when metadata-mode is not available? - if cursor_as_metadata { - CursorMode::Metadata - } else { - CursorMode::Embedded - }, + cursor_mode, source_types, false, None, @@ -184,16 +192,8 @@ impl WaylandCapturableContent { pw_node_id: stream.pipe_wire_node_id(), position: stream.position(), size: stream.size(), - id: if let Some(id) = stream.id() { - Some(id.to_string()) - } else { - None - }, - mapping_id: if let Some(id) = stream.mapping_id() { - Some(id.to_string()) - } else { - None - }, + id: stream.id().map(|id| id.to_string()), + mapping_id: stream.mapping_id().map(|id| id.to_string()), virt: source_type == SourceType::Virtual, cursor_as_metadata, }); @@ -207,16 +207,8 @@ impl WaylandCapturableContent { pw_node_id: stream.pipe_wire_node_id(), position: stream.position(), size: stream.size(), - id: if let Some(id) = stream.id() { - Some(id.to_string()) - } else { - None - }, - mapping_id: if let Some(id) = stream.mapping_id() { - Some(id.to_string()) - } else { - None - }, + id: stream.id().map(|id| id.to_string()), + mapping_id: stream.mapping_id().map(|id| id.to_string()), cursor_as_metadata, }); } diff --git a/src/platform/linux_wayland/capture_stream.rs b/src/platform/linux_wayland/capture_stream.rs index 585cd6d2..1d8855e8 100644 --- a/src/platform/linux_wayland/capture_stream.rs +++ b/src/platform/linux_wayland/capture_stream.rs @@ -23,13 +23,12 @@ use pipewire::{ video::{VideoFormat, VideoInfoRaw}, ParamType, }, - pod::{self, Pod, Property}, + pod::{self, Object, Pod, Property}, sys::{ - spa_buffer, spa_meta_bitmap, spa_meta_cursor, spa_meta_header, - SPA_META_Bitmap, SPA_META_Cursor, SPA_META_Header, - SPA_PARAM_META_size, SPA_PARAM_META_type, + spa_buffer, spa_meta_bitmap, spa_meta_cursor, spa_meta_header, SPA_META_Cursor, + SPA_META_Header, SPA_PARAM_META_size, SPA_PARAM_META_type, SPA_LOG_LEVEL_TRACE, }, - utils::{Direction, Fraction, Rectangle, SpaTypes}, + utils::{ChoiceFlags, Direction, Fraction, Rectangle, SpaTypes}, }, stream::{Stream, StreamFlags, StreamRef, StreamState}, }; @@ -44,39 +43,79 @@ use crate::{ use super::frame::WaylandVideoFrame; +const INVALID_CURSOR_ID: u32 = 0; + +macro_rules! cursor_metadata_size { + ($w:expr, $h:expr) => { + (size_of::() + size_of::() + $w * $h * 4) as i32 + }; +} + +fn serialize_pod_object(obj: Object) -> Result, StreamCreateError> { + let vals: Vec = pod::serialize::PodSerializer::serialize( + std::io::Cursor::new(Vec::new()), + &pod::Value::Object(obj), + ) + .map_err(|e| StreamCreateError::Other(e.to_string()))? + .0 + .into_inner(); + + Ok(vals) +} + +#[derive(Debug, PartialEq)] +struct CursorBitmap { + pub format: VideoFormat, + pub data: Vec, + pub width: u32, + pub height: u32, + pub bytes_per_pixel: usize, +} + #[derive(Default, Debug, PartialEq)] struct PwMetas<'a> { pub header: Option<&'a spa_meta_header>, pub cursor: Option<&'a spa_meta_cursor>, - // TODO: Get the bitmap: https://docs.pipewire.org/video-play_8c-example.html#_a30 - pub bitmap: Option<&'a spa_meta_bitmap>, + pub cursor_bitmap: Option, } impl<'a> PwMetas<'a> { - pub fn from_raw(raw: &'a *mut spa_buffer) -> Self { - let mut self_ = Self::default(); - - unsafe { - let n_metas = (*(*raw)).n_metas; - if n_metas == 0 { - return self_; - } + pub unsafe fn from_raw(raw: &'a *mut spa_buffer) -> Self { + let mut metas = Self::default(); + for meta in std::slice::from_raw_parts((*(*raw)).metas, (*(*raw)).n_metas as usize) { + match meta.type_ { + #[allow(non_upper_case_globals)] + SPA_META_Header => { + metas.header = Some(&*(meta.data as *const spa_meta_header)); + } + #[allow(non_upper_case_globals)] + SPA_META_Cursor => { + let cursor = &*(meta.data as *const spa_meta_cursor); + // Cursor bitmap are only sent when the cursor sprite is different from the previous + if cursor.id != INVALID_CURSOR_ID && cursor.bitmap_offset > 0 { + let bitmap = (cursor as *const spa_meta_cursor) + .byte_offset(cursor.bitmap_offset as isize) + as *const spa_meta_bitmap; + let bitmap_data = std::slice::from_raw_parts( + (bitmap as *const u8).byte_offset((*bitmap).offset as isize), + (*bitmap).size.height as usize * (*bitmap).stride as usize, + ); + metas.cursor_bitmap = Some(CursorBitmap { + format: param::video::VideoFormat((*bitmap).format), + data: bitmap_data.to_vec(), + width: (*bitmap).size.width, + height: (*bitmap).size.height, + bytes_per_pixel: (*bitmap).stride as usize / (*bitmap).size.width as usize, + }); + } - let mut meta_ptr = (*(*raw)).metas; - let metas_end = (*(*raw)).metas.wrapping_add(n_metas as usize); - while meta_ptr != metas_end { - if (*meta_ptr).type_ == SPA_META_Header { - self_.header = Some(&mut *((*meta_ptr).data as *mut spa_meta_header)); - } else if (*meta_ptr).type_ == SPA_META_Cursor { - self_.cursor = Some(&mut *((*meta_ptr).data as *mut spa_meta_cursor)); - } else if (*meta_ptr).type_ == SPA_META_Bitmap { - self_.bitmap = Some(&mut *((*meta_ptr).data as *mut spa_meta_bitmap)) + metas.cursor = Some(cursor); } - meta_ptr = meta_ptr.wrapping_add(1); + _ => {} } } - self_ + metas } } @@ -86,31 +125,13 @@ struct PwDatas<'a> { } impl<'a> PwDatas<'a> { - pub fn from_raw(raw: &'a *mut spa_buffer) -> Vec> { - let mut datas = Vec::new(); - - unsafe { - let n_datas = (*(*raw)).n_datas; - if n_datas == 0 { - return datas; - } - - let mut data_ptr = (*(*raw)).datas; - let datas_end = (*(*raw)).datas.wrapping_add(n_datas as usize); - while data_ptr != datas_end { - if !(*data_ptr).data.is_null() { - datas.push(PwDatas { - data: std::slice::from_raw_parts( - (*data_ptr).data as *mut u8, - (*data_ptr).maxsize as usize, - ), - }); - } - data_ptr = data_ptr.wrapping_add(1); - } - } - - datas + pub unsafe fn from_raw(raw: &'a *mut spa_buffer) -> Vec> { + std::slice::from_raw_parts((*(*raw)).datas, (*(*raw)).n_datas as usize) + .iter() + .map(|data| PwDatas { + data: std::slice::from_raw_parts(data.data as *mut u8, data.maxsize as usize), + }) + .collect::>>() } } @@ -121,6 +142,7 @@ struct WaylandCapturerUD { pub start_time: i64, pub callback: Box) + Send + 'static>, pub should_run: Arc, + pub cursor_bitmap: Option, } pub struct WaylandCaptureStream { @@ -142,19 +164,17 @@ impl WaylandCaptureStream { } fn pod_supported_pixel_formats() -> pod::Property { - pipewire::spa::pod::property!( + pod::property!( format::FormatProperties::VideoFormat, Choice, Enum, Id, - VideoFormat::RGBA, // Big-endian - VideoFormat::RGBx, // Big-endian VideoFormat::BGRx, // Big-endian VideoFormat::BGRA, // Big-endian + VideoFormat::RGBA, // Big-endian + VideoFormat::RGBx, // Big-endian VideoFormat::ABGR, // Big-endian VideoFormat::ARGB, // Big-endian - VideoFormat::xRGB, // Big-endian - VideoFormat::xBGR // Big-endian ) } @@ -191,7 +211,12 @@ impl WaylandCaptureStream { ) } - fn state_changed(_stream: &StreamRef, ud: &mut WaylandCapturerUD, _old: StreamState, new: StreamState) { + fn state_changed( + _stream: &StreamRef, + ud: &mut WaylandCapturerUD, + _old: StreamState, + new: StreamState, + ) { match new { StreamState::Error(e) => { (*ud.callback)(Err(StreamError::Other(e))); @@ -233,55 +258,44 @@ impl WaylandCaptureStream { } }; - let metas_obj = if ud.show_cursor_as_metadata { - pod::object!( - SpaTypes::ObjectParamMeta, - ParamType::Meta, - Property::new( - SPA_PARAM_META_type, - pod::Value::Id(spa::utils::Id(SPA_META_Header)) - ), - Property::new( - SPA_PARAM_META_type, - pod::Value::Id(spa::utils::Id(SPA_META_Cursor)) - ), - Property::new( - SPA_PARAM_META_type, - pod::Value::Id(spa::utils::Id(SPA_META_Bitmap)) - ), - Property::new( - SPA_PARAM_META_size, - pod::Value::Int( - size_of::() as i32 - + size_of::() as i32 - + size_of::() as i32 - ) - ), - ) - } else { - pod::object!( - SpaTypes::ObjectParamMeta, - ParamType::Meta, - Property::new( - SPA_PARAM_META_type, - pod::Value::Id(spa::utils::Id(SPA_META_Header)) - ), - Property::new( - SPA_PARAM_META_size, - pod::Value::Int(size_of::() as i32) - ), - ) - }; - - let metas_values: Vec = pod::serialize::PodSerializer::serialize( - std::io::Cursor::new(Vec::new()), - &pod::Value::Object(metas_obj), - ) - .unwrap() - .0 - .into_inner(); + let mut params = Vec::new(); - let mut params = [pod::Pod::from_bytes(&metas_values).unwrap()]; + let mcursor_obj = pod::object!( + SpaTypes::ObjectParamMeta, + ParamType::Meta, + Property::new( + SPA_PARAM_META_type, + pod::Value::Id(spa::utils::Id(SPA_META_Cursor)) + ), + Property::new( + SPA_PARAM_META_size, + pod::Value::Choice(pod::ChoiceValue::Int(spa::utils::Choice::( + ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Range { + default: cursor_metadata_size!(64, 64), + min: cursor_metadata_size!(1, 1), + max: cursor_metadata_size!(512, 512) + } + ))) + ) + ); + let mcursor_values = serialize_pod_object(mcursor_obj).unwrap(); + params.push(pod::Pod::from_bytes(&mcursor_values).unwrap()); + + let mheader_obj = pod::object!( + SpaTypes::ObjectParamMeta, + ParamType::Meta, + Property::new( + SPA_PARAM_META_type, + pod::Value::Id(spa::utils::Id(SPA_META_Header)) + ), + Property::new( + SPA_PARAM_META_size, + pod::Value::Int(size_of::() as i32) + ), + ); + let mheader_values = serialize_pod_object(mheader_obj).unwrap(); + params.push(pod::Pod::from_bytes(&mheader_values).unwrap()); if let Err(e) = stream.update_params(&mut params) { unsafe { @@ -295,7 +309,8 @@ impl WaylandCaptureStream { } ud.format.parse(param).unwrap(); - println!( // DEBUGGING + println!( + // DEBUGGING "Got pixel format: {} ({:?})", ud.format.format().as_raw(), ud.format.format() @@ -317,13 +332,49 @@ impl WaylandCaptureStream { return; } - let metas = PwMetas::from_raw(&buffer); - let datas = PwDatas::from_raw(&buffer); - if let (Some(header), Some(data)) = (metas.header, datas.iter().next()) { + let (metas, datas) = unsafe { (PwMetas::from_raw(&buffer), PwDatas::from_raw(&buffer)) }; + if let (Some(header), Some(data)) = (metas.header, datas.first()) { if ud.start_time == 0 { ud.start_time = header.pts; } + if metas.cursor_bitmap.is_some() { + ud.cursor_bitmap = metas.cursor_bitmap; + } + + let mut pixel_data = data.data.to_vec(); + 'out: { + if ud.show_cursor_as_metadata { + if let (Some(cursor), Some(bitmap)) = (metas.cursor, ud.cursor_bitmap.as_ref()) + { + if bitmap.format == ud.format.format() { + // TODO: conversion + break 'out; + } + + let mut bmap_iter = bitmap.data.iter(); + // TODO: Accelerate this + for h in cursor.position.y as u32 + ..std::cmp::min( + cursor.position.y as u32 + bitmap.height, + ud.format.size().height, + ) + { + for w in cursor.position.x as usize + ..std::cmp::min( + cursor.position.x as usize + bitmap.width as usize, + ud.format.size().width as usize, + ) + { + for i in 0..bitmap.bytes_per_pixel { + pixel_data[h as usize * w + i] = *bmap_iter.next().unwrap(); + } + } + } + } + } + } + let frame = WaylandVideoFrame { size: crate::prelude::Size { width: ud.format.size().width as f64, @@ -333,7 +384,7 @@ impl WaylandCaptureStream { captured: std::time::Instant::now(), pts: std::time::Duration::from_nanos((header.pts - ud.start_time) as u64), format: ud.format, - data: data.data.to_vec(), + data: pixel_data, }; (*ud.callback)(Ok(StreamEvent::Video(VideoFrame { @@ -351,6 +402,11 @@ impl WaylandCaptureStream { init_tx: &Sender>, ) -> Result<(), StreamCreateError> { pipewire::init(); + unsafe { + // DEBUGGING + pipewire::sys::pw_log_set_level(SPA_LOG_LEVEL_TRACE); + } + let mainloop = MainLoop::new(None).map_err(|e| StreamCreateError::Other(e.to_string()))?; let context = Context::new(&mainloop).map_err(|e| StreamCreateError::Other(e.to_string()))?; @@ -386,6 +442,7 @@ impl WaylandCaptureStream { start_time: 0, callback, should_run: Arc::clone(&should_run), + cursor_bitmap: None, }; let _listener = stream @@ -414,15 +471,8 @@ impl WaylandCaptureStream { Self::pod_supported_framerates(), ); - let stream_param_obj_values: Vec = pod::serialize::PodSerializer::serialize( - std::io::Cursor::new(Vec::new()), - &pod::Value::Object(stream_param_obj), - ) - .map_err(|e| StreamCreateError::Other(e.to_string()))? - .0 - .into_inner(); - - let mut params = [pod::Pod::from_bytes(&stream_param_obj_values).unwrap()]; + let param_obj_values = serialize_pod_object(stream_param_obj)?; + let mut params = [pod::Pod::from_bytes(¶m_obj_values).unwrap()]; stream .connect( @@ -483,8 +533,7 @@ impl WaylandCaptureStream { pub(crate) fn stop(&mut self) -> Result<(), StreamStopError> { if self.should_run.load(Ordering::Acquire) { - self.should_run - .store(false, Ordering::SeqCst); + self.should_run.store(false, Ordering::SeqCst); } if let Some(handle) = self.handle.take() { let _ = handle.join(); @@ -539,7 +588,7 @@ mod tests { }; let mut metas = [spa_meta { type_: SPA_META_Header, - size: std::mem::size_of_val(&meta_header_data) as u32, + size: size_of_val(&meta_header_data) as u32, data: std::ptr::addr_of_mut!(meta_header_data) as *mut c_void, }]; let mut buffer = spa_buffer { @@ -549,19 +598,19 @@ mod tests { datas: std::ptr::null_mut(), }; let buffer_addr = std::ptr::addr_of_mut!(buffer); - let extracted_metas = PwMetas::from_raw(&buffer_addr); + let extracted_metas = unsafe { PwMetas::from_raw(&buffer_addr) }; assert_eq!( extracted_metas, PwMetas { header: Some(&meta_header_data), cursor: None, - bitmap: None + cursor_bitmap: None } ); } #[test] - fn buffer_metas_extraction_header_cursor_bitmap() { + fn buffer_metas_extraction_header_cursor() { let mut meta_header_data = spa_meta_header { flags: 1, offset: 2, @@ -576,31 +625,17 @@ mod tests { hotspot: spa::sys::spa_point { x: 20, y: 22 }, bitmap_offset: 321, }; - let mut meta_bitmap_data = spa_meta_bitmap { - format: 0, - size: spa::sys::spa_rectangle { - width: 0, - height: 0, - }, - stride: 0, - offset: 5, - }; let mut metas = [ spa_meta { type_: SPA_META_Header, - size: std::mem::size_of_val(&meta_header_data) as u32, + size: size_of_val(&meta_header_data) as u32, data: std::ptr::addr_of_mut!(meta_header_data) as *mut c_void, }, spa_meta { type_: SPA_META_Cursor, - size: std::mem::size_of_val(&meta_cursor_data) as u32, + size: size_of_val(&meta_cursor_data) as u32, data: std::ptr::addr_of_mut!(meta_cursor_data) as *mut c_void, }, - spa_meta { - type_: SPA_META_Bitmap, - size: std::mem::size_of_val(&meta_bitmap_data) as u32, - data: std::ptr::addr_of_mut!(meta_bitmap_data) as *mut c_void, - }, ]; let mut buffer = spa_buffer { n_metas: metas.len() as u32, @@ -609,13 +644,13 @@ mod tests { datas: std::ptr::null_mut(), }; let buffer_addr = std::ptr::addr_of_mut!(buffer); - let extracted_metas = PwMetas::from_raw(&buffer_addr); + let extracted_metas = unsafe { PwMetas::from_raw(&buffer_addr) }; assert_eq!( extracted_metas, PwMetas { header: Some(&meta_header_data), cursor: Some(&meta_cursor_data), - bitmap: Some(&meta_bitmap_data) + cursor_bitmap: None } ); } diff --git a/src/platform/linux_wayland/mod.rs b/src/platform/linux_wayland/mod.rs index 0ed0b3f2..403084d3 100644 --- a/src/platform/linux_wayland/mod.rs +++ b/src/platform/linux_wayland/mod.rs @@ -1,6 +1,6 @@ +mod capture_content; mod capture_stream; mod frame; -mod capture_content; pub(crate) use capture_stream::WaylandCaptureAccessToken as ImplCaptureAccessToken; pub(crate) use capture_stream::WaylandCaptureConfig as ImplCaptureConfig; @@ -16,7 +16,6 @@ pub(crate) use capture_content::WaylandCapturableContentFilter as ImplCapturable pub(crate) use capture_content::WaylandCapturableDisplay as ImplCapturableDisplay; pub(crate) use capture_content::WaylandCapturableWindow as ImplCapturableWindow; - #[derive(Clone, Debug)] pub(crate) struct ImplAudioCaptureConfig {} From fc300f866229040dfb5f8f65dd31b599318fb208 Mon Sep 17 00:00:00 2001 From: Marcus Date: Wed, 31 Jul 2024 12:14:03 +0200 Subject: [PATCH 08/10] linux-wayland: feature: bitmap --- src/feature/bitmap/mod.rs | 19 +++++++++++++++++++ src/platform/linux_wayland/capture_stream.rs | 5 +++-- src/platform/linux_wayland/frame.rs | 4 +++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/feature/bitmap/mod.rs b/src/feature/bitmap/mod.rs index 77176479..fe5d25f8 100644 --- a/src/feature/bitmap/mod.rs +++ b/src/feature/bitmap/mod.rs @@ -36,6 +36,9 @@ use windows::Win32::System::WinRT::Direct3D11::IDirect3DDxgiInterfaceAccess; #[cfg(target_os = "windows")] use windows::Win32::Graphics::Direct3D11::D3D11_USAGE_DYNAMIC; +#[cfg(target_os = "linux")] +use pipewire::spa::param::video::VideoFormat; + #[derive(Clone)] struct BitmapPool { free_bitmaps_and_count: Arc>, usize)>>, @@ -526,6 +529,22 @@ impl VideoFrameBitmapInternal for VideoFrame { Err(VideoFrameBitmapError::Other("Failed to lock iosurface".to_string())) } } + #[cfg(target_os = "linux")] + { + match self.impl_video_frame.format.format() { + VideoFormat::BGRA => { + let plane_ptr = VideoFramePlanePtr { + ptr: self.impl_video_frame.data, + width: self.impl_video_frame.size.width as usize, + height: self.impl_video_frame.size.height as usize, + bytes_per_row: self.impl_video_frame.size.width as usize * 4, + }; + + output_mapping(VideoFrameDataCopyPtrs::Bgra8888(plane_ptr)) + } + _ => Err(VideoFrameBitmapError::Other("Invalid pixel format".to_string())), + } + } } } diff --git a/src/platform/linux_wayland/capture_stream.rs b/src/platform/linux_wayland/capture_stream.rs index 1d8855e8..848be001 100644 --- a/src/platform/linux_wayland/capture_stream.rs +++ b/src/platform/linux_wayland/capture_stream.rs @@ -1,6 +1,6 @@ use std::{ cell::RefCell, - ffi::CString, + ffi::{c_void, CString}, mem::size_of, rc::Rc, sync::{ @@ -342,6 +342,7 @@ impl WaylandCaptureStream { ud.cursor_bitmap = metas.cursor_bitmap; } + // Very expensive let mut pixel_data = data.data.to_vec(); 'out: { if ud.show_cursor_as_metadata { @@ -384,7 +385,7 @@ impl WaylandCaptureStream { captured: std::time::Instant::now(), pts: std::time::Duration::from_nanos((header.pts - ud.start_time) as u64), format: ud.format, - data: pixel_data, + data: pixel_data.as_ptr() as *const c_void, }; (*ud.callback)(Ok(StreamEvent::Video(VideoFrame { diff --git a/src/platform/linux_wayland/frame.rs b/src/platform/linux_wayland/frame.rs index cfdf7901..f59d79d5 100644 --- a/src/platform/linux_wayland/frame.rs +++ b/src/platform/linux_wayland/frame.rs @@ -1,3 +1,5 @@ +use std::ffi::c_void; + use pipewire::spa::param::video::VideoInfoRaw; use crate::prelude::VideoCaptureFrame; @@ -8,7 +10,7 @@ pub(crate) struct WaylandVideoFrame { pub(crate) captured: std::time::Instant, pub(crate) pts: std::time::Duration, pub(crate) format: VideoInfoRaw, - pub(crate) data: Vec, // TODO: Optimize + pub(crate) data: *const c_void, } impl VideoCaptureFrame for WaylandVideoFrame { From f24b18a0b4f7e881ce7b87b138b45252d5e91ee5 Mon Sep 17 00:00:00 2001 From: Marcus Date: Wed, 31 Jul 2024 16:13:29 +0200 Subject: [PATCH 09/10] linux-wayland: bitmap: treat BGRx as BGRA --- src/feature/bitmap/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/bitmap/mod.rs b/src/feature/bitmap/mod.rs index fe5d25f8..8264310b 100644 --- a/src/feature/bitmap/mod.rs +++ b/src/feature/bitmap/mod.rs @@ -532,7 +532,7 @@ impl VideoFrameBitmapInternal for VideoFrame { #[cfg(target_os = "linux")] { match self.impl_video_frame.format.format() { - VideoFormat::BGRA => { + VideoFormat::BGRA | VideoFormat::BGRx => { let plane_ptr = VideoFramePlanePtr { ptr: self.impl_video_frame.data, width: self.impl_video_frame.size.width as usize, From cefdce1910227e3c3b3d103317f70b3aad26b7dd Mon Sep 17 00:00:00 2001 From: Marcus Date: Thu, 1 Aug 2024 12:47:58 +0200 Subject: [PATCH 10/10] linux-wayland: disable cursor_as_metadata --- src/platform/linux_wayland/capture_content.rs | 12 +-- src/platform/linux_wayland/capture_stream.rs | 85 +++++++++++-------- 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/src/platform/linux_wayland/capture_content.rs b/src/platform/linux_wayland/capture_content.rs index 32436722..117b22fb 100644 --- a/src/platform/linux_wayland/capture_content.rs +++ b/src/platform/linux_wayland/capture_content.rs @@ -130,11 +130,13 @@ impl WaylandCapturableContent { .await .map_err(|e| CapturableContentError::Other(e.to_string()))?; - let cursor_as_metadata = screencast - .available_cursor_modes() - .await - .map_err(|e| CapturableContentError::Other(e.to_string()))? - .contains(CursorMode::Metadata); + // TODO: Fix dual cursor in kwin when capture monitor and cursor as metadata + // let cursor_as_metadata = screencast + // .available_cursor_modes() + // .await + // .map_err(|e| CapturableContentError::Other(e.to_string()))? + // .contains(CursorMode::Metadata); + let cursor_as_metadata = false; let source_types = Self::source_types_filter(filter) // Some portal implementations freak out when we include supported an not supported source types diff --git a/src/platform/linux_wayland/capture_stream.rs b/src/platform/linux_wayland/capture_stream.rs index 848be001..bcb81a1b 100644 --- a/src/platform/linux_wayland/capture_stream.rs +++ b/src/platform/linux_wayland/capture_stream.rs @@ -173,8 +173,6 @@ impl WaylandCaptureStream { VideoFormat::BGRA, // Big-endian VideoFormat::RGBA, // Big-endian VideoFormat::RGBx, // Big-endian - VideoFormat::ABGR, // Big-endian - VideoFormat::ARGB, // Big-endian ) } @@ -343,38 +341,54 @@ impl WaylandCaptureStream { } // Very expensive - let mut pixel_data = data.data.to_vec(); - 'out: { - if ud.show_cursor_as_metadata { - if let (Some(cursor), Some(bitmap)) = (metas.cursor, ud.cursor_bitmap.as_ref()) - { - if bitmap.format == ud.format.format() { - // TODO: conversion - break 'out; - } - - let mut bmap_iter = bitmap.data.iter(); - // TODO: Accelerate this - for h in cursor.position.y as u32 - ..std::cmp::min( - cursor.position.y as u32 + bitmap.height, - ud.format.size().height, - ) - { - for w in cursor.position.x as usize - ..std::cmp::min( - cursor.position.x as usize + bitmap.width as usize, - ud.format.size().width as usize, - ) - { - for i in 0..bitmap.bytes_per_pixel { - pixel_data[h as usize * w + i] = *bmap_iter.next().unwrap(); - } - } - } - } - } - } + // let mut pixel_data = data.data.to_vec(); + // 'out: { + // if ud.show_cursor_as_metadata { + // if let (Some(cursor), Some(bitmap)) = (metas.cursor, ud.cursor_bitmap.as_ref()) + // { + // if cursor.position.y < 0 + // || cursor.position.x < 0 + // || bitmap.format == ud.format.format() { + // break 'out; + // } + + // TODO: Use cursor.hotspot + // let mut bmap_iter = bitmap.data.iter(); + // let mut h = cursor.position.y as usize; + // let mut height_max = std::cmp::min( + // cursor.position.y as usize + bitmap.height as usize, + // ud.format.size().height as usize, + // ); + // let mut width_max = std::cmp::min( + // cursor.position.x as usize + bitmap.width as usize, + // ud.format.size().width as usize, + // ); + // let stride = 4 * ud.format.size().width as usize; + // while h < height_max { + // let mut w = cursor.position.x as usize; + // while w < width_max { + // if bitmap.bytes_per_pixel != 4 { + // continue; + // } + // let pix_index = h as usize * stride + w as usize * 4; + // let b = *bmap_iter.next().unwrap(); + // let g = *bmap_iter.next().unwrap(); + // let r = *bmap_iter.next().unwrap(); + // let a = *bmap_iter.next().unwrap(); + // if a > 0 { + // pixel_data[pix_index + 0] = b; + // pixel_data[pix_index + 1] = g; + // pixel_data[pix_index + 2] = r; + // pixel_data[pix_index + 3] = a; + // } + // w += 1; + // } + + // h += 1; + // } + // } + // } + // } let frame = WaylandVideoFrame { size: crate::prelude::Size { @@ -385,7 +399,7 @@ impl WaylandCaptureStream { captured: std::time::Instant::now(), pts: std::time::Duration::from_nanos((header.pts - ud.start_time) as u64), format: ud.format, - data: pixel_data.as_ptr() as *const c_void, + data: data.data.as_ptr() as *const c_void, }; (*ud.callback)(Ok(StreamEvent::Video(VideoFrame { @@ -454,6 +468,7 @@ impl WaylandCaptureStream { .register() .map_err(|e| StreamCreateError::Other(e.to_string()))?; + // TODO: Accept DMA buffers let stream_param_obj = pod::object!( spa::utils::SpaTypes::ObjectParamFormat, param::ParamType::EnumFormat,