From c1b9561b892e9d8b4935f01a9187c4f18f11c140 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Fri, 20 Jun 2025 14:35:42 +0200 Subject: [PATCH 01/77] Introduce mxl-sys crate - The crate is supposed to serve as lower level bindings which higher level abstractions can be built upon. Signed-off-by: Pavel Cernohorsky --- rust-bindings/.gitignore | 4 + rust-bindings/Cargo.lock | 309 ++++++++++++++++++ rust-bindings/Cargo.toml | 24 ++ rust-bindings/README.md | 17 + rust-bindings/mxl-sys/Cargo.toml | 10 + rust-bindings/mxl-sys/build.rs | 39 +++ .../mxl-headers-2025-06-17/mxl/dataformat.h | 96 ++++++ .../mxl-sys/mxl-headers-2025-06-17/mxl/flow.h | 307 +++++++++++++++++ .../mxl-headers-2025-06-17/mxl/flowinfo.h | 141 ++++++++ .../mxl-sys/mxl-headers-2025-06-17/mxl/mxl.h | 84 +++++ .../mxl-headers-2025-06-17/mxl/platform.h | 16 + .../mxl-headers-2025-06-17/mxl/rational.h | 23 ++ .../mxl-sys/mxl-headers-2025-06-17/mxl/time.h | 77 +++++ .../mxl-headers-2025-06-17/mxl/version.h | 7 + rust-bindings/mxl-sys/src/lib.rs | 8 + rust-bindings/mxl-sys/tests/simple_test.rs | 11 + 16 files changed, 1173 insertions(+) create mode 100644 rust-bindings/.gitignore create mode 100644 rust-bindings/Cargo.lock create mode 100644 rust-bindings/Cargo.toml create mode 100644 rust-bindings/README.md create mode 100644 rust-bindings/mxl-sys/Cargo.toml create mode 100644 rust-bindings/mxl-sys/build.rs create mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/dataformat.h create mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flow.h create mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flowinfo.h create mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/mxl.h create mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/platform.h create mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/rational.h create mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/time.h create mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/version.h create mode 100644 rust-bindings/mxl-sys/src/lib.rs create mode 100644 rust-bindings/mxl-sys/tests/simple_test.rs diff --git a/rust-bindings/.gitignore b/rust-bindings/.gitignore new file mode 100644 index 000000000..09c22a059 --- /dev/null +++ b/rust-bindings/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +.idea +.vscode +target diff --git a/rust-bindings/Cargo.lock b/rust-bindings/Cargo.lock new file mode 100644 index 000000000..e5868ccb0 --- /dev/null +++ b/rust-bindings/Cargo.lock @@ -0,0 +1,309 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "annotate-snippets" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710e8eae58854cdc1790fcb56cca04d712a17be849eeb81da2a724bf4bae2bc4" +dependencies = [ + "anstyle", + "unicode-width", +] + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "bindgen" +version = "0.72.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f72209734318d0b619a5e0f5129918b848c416e122a3c4ce054e03cb87b726f" +dependencies = [ + "annotate-snippets", + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "libloading" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +dependencies = [ + "cfg-if", + "windows-targets", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mxl-sys" +version = "0.1.0" +dependencies = [ + "bindgen", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "prettyplease" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4307e30089d6fd6aff212f2da3a1f9e32f3223b1f010fb09b7c95f90f3ca1e8" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-width" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" + +[[package]] +name = "windows-targets" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml new file mode 100644 index 000000000..7aa361487 --- /dev/null +++ b/rust-bindings/Cargo.toml @@ -0,0 +1,24 @@ +[workspace] +members = [ + "mxl-sys", +] + +resolver = "2" + +[workspace.package] +edition = "2024" +publish = false +version = "0.1.0" +license = "Apache-2.0" +license-file = "../LICENSE.txt" + +[workspace.dependencies] +bindgen = { version = "0.72", features = ["experimental"] } +dlopen2 = "0.8" +futures = "0.3" +tracing = { version = "0.1", features = ["log"] } +tracing-subscriber = { version = "0.3", features = ["registry"] } + +[profile.release-with-debug] +debug = true +inherits = "release" diff --git a/rust-bindings/README.md b/rust-bindings/README.md new file mode 100644 index 000000000..f122d98b3 --- /dev/null +++ b/rust-bindings/README.md @@ -0,0 +1,17 @@ +# Rust bindings for DMF MXL + +## Code guidelines + +- Use `rustfmt` in it's default settings for code formatting. +- The `cargo clippy` should be always clean. + +## Building + +- `cargo build` + +## TODO + +- Get rid of the headers copy. Use the main headers as part of the build process. +- Change the tests so they can use libraries build from the main repo. +- Setup CI/CD. +- Extend the functionality. diff --git a/rust-bindings/mxl-sys/Cargo.toml b/rust-bindings/mxl-sys/Cargo.toml new file mode 100644 index 000000000..308060e71 --- /dev/null +++ b/rust-bindings/mxl-sys/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mxl-sys" +edition.workspace = true +publish.workspace = true +version.workspace = true + +[dependencies] + +[build-dependencies] +bindgen.workspace = true diff --git a/rust-bindings/mxl-sys/build.rs b/rust-bindings/mxl-sys/build.rs new file mode 100644 index 000000000..7e6d25188 --- /dev/null +++ b/rust-bindings/mxl-sys/build.rs @@ -0,0 +1,39 @@ +use std::env; +use std::path::PathBuf; + +fn main() { + let headers_dir = "mxl-headers-2025-06-17"; + let headers = [ + "mxl/dataformat.h", + "mxl/flow.h", + "mxl/flowinfo.h", + "mxl/mxl.h", + "mxl/platform.h", + "mxl/rational.h", + "mxl/time.h", + "mxl/version.h", + ]; + + let manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("failed to get current directory"); + let includes_dir = format!("{manifest_dir}/{headers_dir}"); + println!("cargo:include={includes_dir}"); + + let bindings = bindgen::builder() + .clang_arg(format!("-I{includes_dir}")) + .headers( + headers + .iter() + .map(|val| format!("{includes_dir}/{val}")) + .collect::>(), + ) + .derive_default(true) + .derive_debug(true) + .prepend_enum_name(false) + .generate() + .unwrap(); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("Could not write bindings"); +} diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/dataformat.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/dataformat.h new file mode 100644 index 000000000..f30565689 --- /dev/null +++ b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/dataformat.h @@ -0,0 +1,96 @@ +#pragma once + +#ifdef __cplusplus +extern "C" +{ +#endif + /** + * Source and flow data formats as defined by AMWA NMOS IS-04, excluding `urn:x-nmos:format:data.event`. + */ + typedef enum mxlDataFormat + { + MXL_DATA_FORMAT_UNSPECIFIED, + MXL_DATA_FORMAT_VIDEO, + MXL_DATA_FORMAT_AUDIO, + MXL_DATA_FORMAT_DATA, + MXL_DATA_FORMAT_MUX, + } mxlDataFormat; + + /** + * Return whether the specified format is valid. + * \param[in] format the mxlDataFormat of interest. + * \return 1 if the format specified in \p format is valid, otherwise 0. + */ + inline int mxlIsValidDataFormat(int format) + { + switch (format) + { + case MXL_DATA_FORMAT_VIDEO: + case MXL_DATA_FORMAT_AUDIO: + case MXL_DATA_FORMAT_DATA: + case MXL_DATA_FORMAT_MUX: + return 1; + + default: + return 0; + } + } + + /** + * Return whether the specified format is supported by MXL. + * \param[in] format the mxlDataFormat of interest. + * \return 1 if the format specified in \p format is supported, otherwise 0. + */ + inline int mxlIsSupportedDataFormat(int format) + { + switch (format) + { + case MXL_DATA_FORMAT_VIDEO: + case MXL_DATA_FORMAT_AUDIO: + case MXL_DATA_FORMAT_DATA: + return 1; + + default: + return 0; + } + } + + /** + * Return whether the specified format is operating in discrete grains. + * \param[in] format the mxlDataFormat of interest. + * \return 1 if the format specified in \p format is operating with + * continuous samples, otherwise 0. + */ + inline int mxlIsDiscreteDataFormat(int format) + { + switch (format) + { + case MXL_DATA_FORMAT_VIDEO: + case MXL_DATA_FORMAT_DATA: + return 1; + + default: + return 0; + } + } + + /** + * Return whether the specified format is operating in continuous samples. + * \param[in] format the mxlDataFormat of interest. + * \return 1 if the format specified in \p format is operating with + * continuous samples, otherwise 0. + */ + inline int mxlIsContinuousDataFormat(int format) + { + switch (format) + { + case MXL_DATA_FORMAT_AUDIO: + return 1; + + default: + return 0; + } + } +#ifdef __cplusplus +} +#endif diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flow.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flow.h new file mode 100644 index 000000000..2d1ace473 --- /dev/null +++ b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flow.h @@ -0,0 +1,307 @@ +#pragma once + +#ifdef __cplusplus +# include +# include +#else +# include +# include +#endif + +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* + * A grain can be marked as invalid for multiple reasons. for example, an input application may have + * timed out before receiving a grain in time, etc. Writing grain marked as invalid is the proper way + * to make the ringbuffer whilst letting consumers know that the grain is invalid. A consumer + * may choose to repeat the previous grain, insert silence, etc. + */ +#define GRAIN_FLAG_INVALID 0x00000001 // 1 << 0. + + /** + * The payload location of the grain + */ + typedef enum PayloadLocation + { + PAYLOAD_LOCATION_HOST_MEMORY = 0, + PAYLOAD_LOCATION_DEVICE_MEMORY = 1, + } PayloadLocation; + + /** + * A helper type used to describe consecutive sequences of bytes in memory. + */ + typedef struct BufferSlice + { + /** A pointer referring to the beginning of the slice. */ + void const* pointer; + /** The number of bytes that make up this slice. */ + size_t size; + } BufferSlice; + + /** + * A helper type used to describe consecutive sequences of mutable bytes in + * memory. + */ + typedef struct MutableBufferSlice + { + /** A pointer referring to the beginning of the slice. */ + void* pointer; + /** The number of bytes that make up this slice. */ + size_t size; + } MutableBufferSlice; + + /** + * A helper type used to describe consecutive sequences of bytes + * in a ring buffer that may potentially straddle the wrapraround + * point of the buffer. + */ + typedef struct WrappedBufferSlice + { + BufferSlice fragments[2]; + } WrappedBufferSlice; + + /** + * A helper type used to describe consecutive sequences of mutable bytes in + * a ring buffer that may potentially straddle the wrapraround point of the + * buffer. + */ + typedef struct MutableWrappedBufferSlice + { + MutableBufferSlice fragments[2]; + } MutableWrappedBufferSlice; + + /** + * A helper type used to describe consecutive sequences of bytes + * in memory in consecutive ring buffers separated by the specified + * stride of bytes. + */ + typedef struct WrappedMultiBufferSlice + { + WrappedBufferSlice base; + + /** + * The stride in bytes to get from a position in one buffer + * to the same position in the following buffer. + */ + size_t stride; + /** + * The number of buffers following the base buffer. + */ + size_t count; + } WrappedMultiBufferSlice; + + /** + * A helper type used to describe consecutive sequences of mutable bytes in + * memory in consecutive ring buffers separated by the specified stride of + * bytes. + */ + typedef struct MutableWrappedMultiBufferSlice + { + MutableWrappedBufferSlice base; + + /** + * The stride in bytes to get from a position in one buffer + * to the same position in the following buffer. + */ + size_t stride; + /** + * The number of buffers following the base buffer. + */ + size_t count; + } MutableWrappedMultiBufferSlice; + + + + typedef struct GrainInfo + { + /// Version of the structure. The only currently supported value is 1 + uint32_t version; + /// Size of the structure + uint32_t size; + /// Grain flags. + uint32_t flags; + /// Payload location + PayloadLocation payloadLocation; + /// Device index (if payload is in device memory). -1 if on host memory. + int32_t deviceIndex; + /// Size in bytes of the complete payload of a grain + uint32_t grainSize; + /// How many bytes in the grain are currently valid (commited). This is typically used when writing slices. + /// A grain is complete when commitedSize == grainSize + uint32_t commitedSize; + /// User data space + uint8_t userData[4068]; + } GrainInfo; + + typedef struct mxlFlowReader_t* mxlFlowReader; + typedef struct mxlFlowWriter_t* mxlFlowWriter; + + /// + /// Create a flow using a json flow definition + /// + /// \param[in] instance The mxl instance created using mxlCreateInstance + /// \param[in] flowDef A flow definition in the NMOS Flow json format. The flow ID is read from the field of this json object. + /// \param[in] options Additional options (undefined). \todo Specify and used the additional options. + /// \param[out] info A pointer to a FlowInfo structure. If not nullptr, this structure will be updated with the flow information after the flow is + /// created. +MXL_EXPORT + mxlStatus mxlCreateFlow(mxlInstance instance, char const* flowDef, char const* options, FlowInfo* info); + + MXL_EXPORT + mxlStatus mxlDestroyFlow(mxlInstance instance, char const* flowId); + + MXL_EXPORT + mxlStatus mxlCreateFlowReader(mxlInstance instance, char const* flowId, char const* options, mxlFlowReader* reader); + + MXL_EXPORT + mxlStatus mxlReleaseFlowReader(mxlInstance instance, mxlFlowReader reader); + + MXL_EXPORT + mxlStatus mxlCreateFlowWriter(mxlInstance instance, char const* flowId, char const* options, mxlFlowWriter* writer); + + MXL_EXPORT + mxlStatus mxlReleaseFlowWriter(mxlInstance instance, mxlFlowWriter writer); + + /** + * Get a copy of the current descriptive header of a Flow + * + * \param[in] reader A valid flow reader + * \param[out] info A valid pointer to a FlowInfo structure. on return, the structure will be updated with a copy of the current flow info value. + * \return The result code. \see mxlStatus + */ +MXL_EXPORT + mxlStatus mxlFlowReaderGetInfo(mxlFlowReader reader, FlowInfo* info); + + /** + * Accessors for a flow grain at a specific index + * + * \param[in] reader A valid discrete flow reader. + * \param[in] index The index of the grain to obtain + * \param[in] timeoutNs How long should we wait for the grain (in nanoseconds) + * \param[out] grain The requested GrainInfo structure. + * \param[out] payload The requested grain payload. + * \return The result code. \see mxlStatus + * \note Please note that this function can only be called on readers that + * operate on discrete flows. Any attempt to call this function on a + * reader that operates on another type of flow will result in an + * error. + */ +MXL_EXPORT + mxlStatus mxlFlowReaderGetGrain(mxlFlowReader reader, uint64_t index, uint64_t timeoutNs, GrainInfo* grain, + uint8_t** payload); + + /** + * Non-blocking accessors for a flow grain at a specific index + * + * \param[in] reader A valid flow reader + * \param[in] index The index of the grain to obtain + * \param[out] grain The requested GrainInfo structure. + * \param[out] payload The requested grain payload. + * \return The result code. \see mxlStatus + * \note Please note that this function can only be called on readers that + * operate on discrete flows. Any attempt to call this function on a + * reader that operates on another type of flow will result in an + * error. + */ +MXL_EXPORT + mxlStatus mxlFlowReaderGetGrainNonBlocking(mxlFlowReader reader, uint64_t index, GrainInfo* grain, + uint8_t** payload); + + /** + * Open a grain for mutation. The flow writer will remember which index is currently opened. Before opening a new grain + * for mutation, the user must either cancel the mutation using mxlFlowWriterCancelGrain or mxlFlowWriterCommitGrain. + * + * \todo Allow operating on multiple grains simultaneously, by making this function return a handle that has to be passed + * to mxlFlowWriterCommitGrain or mxlFlowWriterCancelGrain to identify the grain the call refers to. + * + * \param[in] writer A valid flow writer + * \param[in] index The index of the grain to obtain + * \param[out] grainInfo The requested GrainInfo structure. + * \param[out] payload The requested grain payload. + * \return The result code. \see mxlStatus + * \note Please note that this function can only be called on writers that + * operate on discrete flows. Any attempt to call this function on a + * writer that operates on another type of flow will result in an + * error. + */ +MXL_EXPORT + mxlStatus mxlFlowWriterOpenGrain(mxlFlowWriter writer, uint64_t index, GrainInfo* grainInfo, + uint8_t** payload); + + /** + * + * \param[in] writer A valid flow writer + */ +MXL_EXPORT + mxlStatus mxlFlowWriterCancelGrain(mxlFlowWriter writer); + + /** + * Inform mxl that a user is done writing the grain that was previously opened. This will in turn signal all readers waiting on the ringbuffer + * that a new grain is available. The graininfo flags field in shared memory will be updated based on grain->flags This will increase the head + * and potentially the tail IF this grain is the new head. + * + * \return The result code. \see mxlStatus + */ +MXL_EXPORT + mxlStatus mxlFlowWriterCommitGrain(mxlFlowWriter writer, GrainInfo const* grain); + + + /** + * Accessor for a specific set of samples across all channels starting at a + * specific index. + * + * \param[in] index The head index of the samples to obtain. + * \param[in] count The number of samples to obtain. + * \param[out] payloadBuffersSlices A pointer to a wrapped multi buffer + * slice that represents the requested range across all channel + * buffers. + * + * \return A status code describing the outcome of the call. + * \note No guarantees are made as to how long the caller may + * safely hang on to the returned range of samples without the + * risk of these samples being overwritten. + */ + MXL_EXPORT + mxlStatus mxlFlowReaderGetSamples(mxlFlowReader reader, uint64_t index, size_t count, WrappedMultiBufferSlice* payloadBuffersSlices); + + /** + * Open a specific set of mutable samples across all channels starting at a + * specific index for mutation. + * + * \param[in] index The head index of the samples that will be mutated. + * \param[in] count The number of samples in each channel that will be + * mutated. + * \param[out] payloadBuffersSlices A pointer to a mutable wrapped multi + * buffer slice that represents the requested range across all channel + * buffers. + * + * \return A status code describing the outcome of the call. + */ + MXL_EXPORT + mxlStatus mxlFlowWriterOpenSamples(mxlFlowWriter writer, uint64_t index, size_t count, MutableWrappedMultiBufferSlice* payloadBuffersSlices); + + /** + * Cancel the mutation of the previously opened range of samples. + * \param[in] writer A valid flow writer + * \return The result code. \see mxlStatus + */ + MXL_EXPORT + mxlStatus mxlFlowWriterCancelSamples(mxlFlowWriter writer); + + /** + * Inform mxl that a user is done writing the sample range that was previously opened. + * + * \param[in] writer A valid flow writer + * \return The result code. \see mxlStatus + */ + MXL_EXPORT + mxlStatus mxlFlowWriterCommitSamples(mxlFlowWriter writer); +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flowinfo.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flowinfo.h new file mode 100644 index 000000000..65d7fceb2 --- /dev/null +++ b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flowinfo.h @@ -0,0 +1,141 @@ +#pragma once + +#ifdef __cplusplus +# include +#else +# include +#endif + +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + /** + * Metadata about media a flow that is independent of the data format of the + * flow and thus common to all flows handled by MXL. + */ + typedef struct CommonFlowInfo + { + /** The flow UUID. This should be identical to the {flowId} path component. */ + uint8_t id[16]; + + + /** The last time a producer wrote to the flow in nanoseconds since the epoch. */ + uint64_t lastWriteTime; + + /** The last time a consumer read from the flow in nanoseconds since the epoch. */ + uint64_t lastReadTime; + + + /** + * The data format of this flow. + * \see mxlDataFormat + */ + uint32_t format; + + /** No flags defined yet. */ + uint32_t flags; + + /** Reserved space for future extensions. */ + uint8_t reserved[80]; + } CommonFlowInfo; + + typedef struct DiscreteFlowInfo + { + /** + * The number of grains per second expressed as a rational. + * For VIDEO and DATA this value must match the 'grain_rate' found in the flow descriptor. + */ + Rational grainRate; + + /** + * How many grains in the ring buffer. This should be identical to the number of entries in the {mxlDomain}/{flowId}/grains/ folder. + * Accessing the shared memory section for that specific grain should be predictable. + */ + uint32_t grainCount; + + /** + * 32 bit word used syncronization between a writer and multiple readers. This value can be used by futexes. + * When a FlowWriter commits some data (a grain, a slice, etc) it will increment this value and then wake all FlowReaders waiting on this + * memory address. + */ + uint32_t syncCounter; + + /** The current head index of the ringbuffer. */ + uint64_t headIndex; + + /** Reserved space for future extensions. */ + uint8_t reserved[96]; + } DiscreteFlowInfo; + + typedef struct ContinuousFlowInfo + { + /** + * The number of samples per second in this continuous flow. + * For AUDIO flows this value must match the 'sample_rate' found in the flow descriptor. + */ + Rational sampleRate; + + /** + * The number of channels in this flow. + * A dedicated ring buffer is provided for each channel. + */ + uint32_t channelCount; + + /** + * The number of samples in each of the ring buffers. + */ + uint32_t bufferLength; + + /** + * The largest expected batch size in samples, in which new data is written to this this flow by its producer. + * This value must be less than half of the buffer length. + */ + uint32_t commitBatchSize; + + /** + * The largest expected batch size in samples, at which availability of new data is signaled to waiting consumers. + * This must be a multiple of the commit batch size greater or equal to 1. + * \todo Will quite probably be obsoleted by new timing model. + */ + uint32_t syncBatchSize; + + /** The current head index within the per channel ring buffers. */ + uint64_t headIndex; + + /** Reserved space for future extensions. */ + uint8_t reserved[88]; + } ContinuousFlowInfo; + + /** + * Binary structure stored in the Flow shared memory segment. + * The flow shared memory will be located in {mxlDomain}/{flowId} + * where {mxlDomain} is a filesystem location available to the application + */ + typedef struct FlowInfo + { + /** Version of this structure. The only currently supported value is 1 */ + uint32_t version; + + /** The total size of this structure */ + uint32_t size; + + CommonFlowInfo common; + + /** Format specific header data. */ + union + { + DiscreteFlowInfo discrete; + ContinuousFlowInfo continuous; + }; + + /** User data space. */ + uint8_t userData[3840]; + } FlowInfo; + +#ifdef __cplusplus +} +#endif diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/mxl.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/mxl.h new file mode 100644 index 000000000..9807bb7a7 --- /dev/null +++ b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/mxl.h @@ -0,0 +1,84 @@ +#pragma once + +#ifdef __cplusplus +# include +#else +# include +#endif + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + + /// MXL SDK Status codes. + typedef enum mxlStatus + { + MXL_STATUS_OK, + MXL_ERR_UNKNOWN, + MXL_ERR_FLOW_NOT_FOUND, + MXL_ERR_OUT_OF_RANGE_TOO_LATE, + MXL_ERR_OUT_OF_RANGE_TOO_EARLY, + MXL_ERR_INVALID_FLOW_READER, + MXL_ERR_INVALID_FLOW_WRITER, + MXL_ERR_TIMEOUT, + MXL_ERR_INVALID_ARG, + MXL_ERR_CONFLICT, + } mxlStatus; + + /// MXL SDK Semantic versionning structure. + typedef struct mxlVersionType + { + uint16_t major; + uint16_t minor; + uint16_t bugfix; + uint16_t build; + } mxlVersionType; + + /// + /// Accessor for the version of the MXL SDK. + /// \param out_version Pointer to a mxlVersionType structure to be filled with the version information. + /// \return MXL_STATUS_OK if the version was successfully retrieved, MXL_ERR_INVALID_ARG if the pointer passed was NULL. + /// + MXL_EXPORT + mxlStatus mxlGetVersion(mxlVersionType* out_version); + + /** An opaque type representing an MXL instance. */ + typedef struct mxlInstance_t* mxlInstance; + + /// + /// Create a new MXL instance for a specific domain. + /// + /// \param in_mxlDomain The domain is the directory where the MXL ringbuffers files are stored. It should live on a tmpfs filesystem. + /// \param in_options Optional JSON string containing additional SDK options. Currently not used. + /// \return A pointer to the MXL instance or NULL if the instance could not be created. + /// + MXL_EXPORT + mxlInstance mxlCreateInstance(char const* in_mxlDomain, char const* in_options); + + /// + /// Iterates over all flows in the MXL domain and deletes any flows that are + /// no longer active (in other words, no readers or writers are using them. This typically + /// happens when an application creating flows writers crashes or exits without cleaning up. + /// A flow is considered active if a shared advisory lock is held on the data file of the flow. + /// This function is automatically called when the instance is created but should be called periodically + /// on a long running application to clean up any flows that are no longer active. + /// + MXL_EXPORT + mxlStatus mxlGarbageCollectFlows(mxlInstance in_instance); + + /// + /// Destroy the MXL instance. This will also release all flows readers/writers associated with the instance. + /// + /// \param in_instance The MXL instance to destroy. + /// \return MXL_STATUS_OK if the instance was successfully destroyed, MXL_ERR_INVALID_ARG if the pointer passed was NULL. MXL_ERR_UNKNOWN if an + /// error occurred during destruction. + /// + MXL_EXPORT + mxlStatus mxlDestroyInstance(mxlInstance in_instance); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/platform.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/platform.h new file mode 100644 index 000000000..0d6f96ada --- /dev/null +++ b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/platform.h @@ -0,0 +1,16 @@ +#pragma once + +#if defined(__GNUC__) || defined(__clang__) +# define MXL_EXPORT __attribute__((visibility("default"))) +#else +# define MXL_EXPORT +#endif + +// TODO: Tailor these more to specific language statdard levels +#ifdef __cplusplus +# define MXL_NODISCARD [[nodiscard]] +# define MXL_CONSTEXPR constexpr +#else +# define MXL_NODISCARD +# define MXL_CONSTEXPR inline +#endif diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/rational.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/rational.h new file mode 100644 index 000000000..9cb89bd09 --- /dev/null +++ b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/rational.h @@ -0,0 +1,23 @@ +#pragma once + +#ifdef __cplusplus +# include +#else +# include +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef struct Rational + { + int64_t numerator; + int64_t denominator; + } Rational; + +#ifdef __cplusplus +} +#endif + diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/time.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/time.h new file mode 100644 index 000000000..324467e52 --- /dev/null +++ b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/time.h @@ -0,0 +1,77 @@ +#pragma once + +#ifdef __cplusplus +# include +#else +# include +#endif + +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define MXL_UNDEFINED_INDEX UINT64_MAX + + /** + * Get the current head index based on the current system TAI time + * Index 0 is defined to be index at the beginning of the epoch as defined by SMPTE ST 2059 + * + * \param[in] editRate The edit rate of the Flow + * \return The head index or MXL_UNDEFINED_INDEX if editRate is null or invalid + */ +MXL_EXPORT + uint64_t mxlGetCurrentHeadIndex(Rational const* editRate); + + /** + * Utility method to help compute how many nanoseconds we need to wait until the beginning of the specified index + * \param[in] index The head to wait for + * \param[in] editRate The edit rate of the Flow + * \return How many nanoseconds to wait or MXL_UNDEFINED_INDEX if editRate is null or invalid + */ +MXL_EXPORT + uint64_t mxlGetNsUntilHeadIndex(uint64_t index, Rational const* editRate); + + /** + * Get the current head index based on the user provided timespec + * Index 0 is defined to be index at the beginning of the epoch as defined by SMPTE ST 2059 + * + * \param[in] editRate The edit rate of the Flow + * \param[in] timestamp The time stamp in nanoseconds since the epoch. + * \return The head index or MXL_UNDEFINED_INDEX if editRate is null or invalid + */ +MXL_EXPORT + uint64_t mxlTimestampToHeadIndex(Rational const* editRate, uint64_t timestamp); + + /** + * Get the current timestamp based on the user provided head index. + * Index 0 is defined to be index at the beginning of the epoch as defined by SMPTE ST 2059 + * + * \param[in] editRate The edit rate of the Flow + * \param[in] index The head index of the flow. + * \return The time stamp in nanoseconds since the epoch or MXL_UNDEFINED_INDEX if editRate is null or invalid + */ +MXL_EXPORT + uint64_t mxlHeadIndexToTimestamp(Rational const* editRate, uint64_t index); + + /** + * Sleep for a specific amount of time. + * \param[in] ns How long to sleep for, in nanoseconds. + */ +MXL_EXPORT + void mxlSleepForNs(uint64_t ns); + + /** + * Get the current time using the most appropriate clock for the platform. + * + * \return The current time in nanoseconds since the epoch. + */ +MXL_EXPORT + uint64_t mxlGetTime(); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/version.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/version.h new file mode 100644 index 000000000..ba2442a54 --- /dev/null +++ b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/version.h @@ -0,0 +1,7 @@ +#pragma once + +#define MXL_VERSION_MAJOR 0 +#define MXL_VERSION_MINOR 6 +#define MXL_VERSION_PATCH 0 +#define MXL_VERSION_BUILD 0 +#define MXL_VERSION "0.6.0.0" diff --git a/rust-bindings/mxl-sys/src/lib.rs b/rust-bindings/mxl-sys/src/lib.rs new file mode 100644 index 000000000..666331b0d --- /dev/null +++ b/rust-bindings/mxl-sys/src/lib.rs @@ -0,0 +1,8 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(missing_docs)] +// Suppress expected warnings from bindgen-generated code. +// See https://github.com/rust-lang/rust-bindgen/issues/1651. +#![allow(deref_nullptr)] +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/rust-bindings/mxl-sys/tests/simple_test.rs b/rust-bindings/mxl-sys/tests/simple_test.rs new file mode 100644 index 000000000..c617fe566 --- /dev/null +++ b/rust-bindings/mxl-sys/tests/simple_test.rs @@ -0,0 +1,11 @@ +#[test] +fn there_is_bindgen_generated_code() { + let mxl_version = mxl_sys::mxlVersionType { + major: 3, + minor: 2, + bugfix: 1, + ..Default::default() + }; + + println!("mxl_version: {:?}", mxl_version); +} From 7b46ad29c1b582d8851e8895156199c8c61bbe1d Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Fri, 20 Jun 2025 15:25:39 +0200 Subject: [PATCH 02/77] Add first skeleton for "middle layer" binding - This layer is supposed to more or less reflect the structure of the MXL library itself and should not yet use async. Signed-off-by: Pavel Cernohorsky --- rust-bindings/.gitattributes | 1 + rust-bindings/Cargo.lock | 232 ++++++++++++++- rust-bindings/Cargo.toml | 7 +- rust-bindings/README.md | 9 +- rust-bindings/mxl/Cargo.toml | 15 + rust-bindings/mxl/src/lib.rs | 385 +++++++++++++++++++++++++ rust-bindings/mxl/tests/basic_tests.rs | 54 ++++ 7 files changed, 697 insertions(+), 6 deletions(-) create mode 100644 rust-bindings/.gitattributes create mode 100644 rust-bindings/mxl/Cargo.toml create mode 100644 rust-bindings/mxl/src/lib.rs create mode 100644 rust-bindings/mxl/tests/basic_tests.rs diff --git a/rust-bindings/.gitattributes b/rust-bindings/.gitattributes new file mode 100644 index 000000000..383338e04 --- /dev/null +++ b/rust-bindings/.gitattributes @@ -0,0 +1 @@ +Cargo.lock -diff diff --git a/rust-bindings/Cargo.lock b/rust-bindings/Cargo.lock index e5868ccb0..21f575c3b 100644 --- a/rust-bindings/Cargo.lock +++ b/rust-bindings/Cargo.lock @@ -80,6 +80,29 @@ dependencies = [ "libloading", ] +[[package]] +name = "dlopen2" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b54f373ccf864bf587a89e880fb7610f8d73f3045f13580948ccbcaff26febff" +dependencies = [ + "dlopen2_derive", + "libc", + "once_cell", + "winapi", +] + +[[package]] +name = "dlopen2_derive" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "788160fb30de9cdd857af31c6a2675904b16ece8fc2737b2c7127ba368c9d0f4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.15.0" @@ -101,6 +124,12 @@ dependencies = [ "either", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.174" @@ -123,6 +152,15 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "memchr" version = "2.7.5" @@ -135,6 +173,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "mxl" +version = "0.1.0" +dependencies = [ + "dlopen2", + "mxl-sys", + "thiserror", + "tracing", + "tracing-subscriber", +] + [[package]] name = "mxl-sys" version = "0.1.0" @@ -152,6 +201,34 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + [[package]] name = "prettyplease" version = "0.2.34" @@ -188,8 +265,17 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -200,9 +286,15 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.5", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.5" @@ -215,12 +307,27 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + [[package]] name = "syn" version = "2.0.103" @@ -232,6 +339,97 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + [[package]] name = "unicode-ident" version = "1.0.18" @@ -244,6 +442,34 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-targets" version = "0.53.2" diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml index 7aa361487..77df3d4a3 100644 --- a/rust-bindings/Cargo.toml +++ b/rust-bindings/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ - "mxl-sys", + "mxl", + "mxl-sys", ] resolver = "2" @@ -15,9 +16,11 @@ license-file = "../LICENSE.txt" [workspace.dependencies] bindgen = { version = "0.72", features = ["experimental"] } dlopen2 = "0.8" +# Will be used later, when we get to higher level streams based interfaces. futures = "0.3" +thiserror = "2.0.12" tracing = { version = "0.1", features = ["log"] } -tracing-subscriber = { version = "0.3", features = ["registry"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "std"] } [profile.release-with-debug] debug = true diff --git a/rust-bindings/README.md b/rust-bindings/README.md index f122d98b3..24d9051bc 100644 --- a/rust-bindings/README.md +++ b/rust-bindings/README.md @@ -1,9 +1,16 @@ # Rust bindings for DMF MXL -## Code guidelines +## Goals + +- Hide all the unsafe stuff inside these bindings. +- Provide more Rust-native like experience (async API based on `futures::stream` and + `futures::sink`?). + +## Code Guidelines - Use `rustfmt` in it's default settings for code formatting. - The `cargo clippy` should be always clean. +- Try to avoid adding more dependencies, unless really necessary. ## Building diff --git a/rust-bindings/mxl/Cargo.toml b/rust-bindings/mxl/Cargo.toml new file mode 100644 index 000000000..fef324986 --- /dev/null +++ b/rust-bindings/mxl/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "mxl" +edition.workspace = true +publish.workspace = true +version.workspace = true + +[dependencies] +mxl-sys = { path = "../mxl-sys" } + +dlopen2.workspace = true +thiserror.workspace = true +tracing.workspace = true + +[dev-dependencies] +tracing-subscriber.workspace = true diff --git a/rust-bindings/mxl/src/lib.rs b/rust-bindings/mxl/src/lib.rs new file mode 100644 index 000000000..ada237b28 --- /dev/null +++ b/rust-bindings/mxl/src/lib.rs @@ -0,0 +1,385 @@ +use std::ffi::CString; +use std::path::Path; +use std::time::Duration; + +use dlopen2::wrapper::{Container, WrapperApi}; + +#[derive(Debug, thiserror::Error)] +pub enum MxlError { + #[error("Unknown error")] + Unknown, + #[error("Flow not found")] + FlowNotFound, + #[error("Out of range - too late")] + OutOfRangeTooLate, + #[error("Out of range - too early")] + OutOfRangeTooEarly, + #[error("Invalid flow reader")] + InvalidFlowReader, + #[error("Invalid flow writer")] + InvalidFlowWriter, + #[error("Timeout")] + Timeout, + #[error("Invalid argument")] + InvalidArg, + #[error("Conflict")] + Conflict, + /// The error is not defined in the MXL API, but it is used to wrap other errors. + #[error("Other error: {0}")] + Other(String), +} + +fn status_to_result(status: mxl_sys::mxlStatus) -> Result<(), MxlError> { + match status { + mxl_sys::MXL_STATUS_OK => Ok(()), + mxl_sys::MXL_ERR_UNKNOWN => Err(MxlError::Unknown), + mxl_sys::MXL_ERR_FLOW_NOT_FOUND => Err(MxlError::FlowNotFound), + mxl_sys::MXL_ERR_OUT_OF_RANGE_TOO_LATE => Err(MxlError::OutOfRangeTooLate), + mxl_sys::MXL_ERR_OUT_OF_RANGE_TOO_EARLY => Err(MxlError::OutOfRangeTooEarly), + mxl_sys::MXL_ERR_INVALID_FLOW_READER => Err(MxlError::InvalidFlowReader), + mxl_sys::MXL_ERR_INVALID_FLOW_WRITER => Err(MxlError::InvalidFlowWriter), + mxl_sys::MXL_ERR_TIMEOUT => Err(MxlError::Timeout), + mxl_sys::MXL_ERR_INVALID_ARG => Err(MxlError::InvalidArg), + mxl_sys::MXL_ERR_CONFLICT => Err(MxlError::Conflict), + _ => Err(MxlError::Unknown), + } +} + +#[derive(WrapperApi)] +pub struct MxlApi { + #[dlopen2_name = "mxlGetVersion"] + mxl_get_version: + unsafe extern "C" fn(out_version: *mut mxl_sys::mxlVersionType) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlCreateInstance"] + mxl_create_instance: unsafe extern "C" fn( + in_mxlDomain: *const std::os::raw::c_char, + in_options: *const std::os::raw::c_char, + ) -> mxl_sys::mxlInstance, + #[dlopen2_name = "mxlGarbageCollectFlows"] + mxl_garbage_collect_flows: + unsafe extern "C" fn(in_instance: mxl_sys::mxlInstance) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlDestroyInstance"] + mxl_destroy_instance: + unsafe extern "C" fn(in_instance: mxl_sys::mxlInstance) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlCreateFlow"] + mxl_create_flow: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + flowDef: *const std::os::raw::c_char, + options: *const std::os::raw::c_char, + info: *mut mxl_sys::FlowInfo, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlDestroyFlow"] + mxl_destroy_flow: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + flowId: *const std::os::raw::c_char, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlCreateFlowReader"] + mxl_create_flow_reader: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + flowId: *const std::os::raw::c_char, + options: *const std::os::raw::c_char, + reader: *mut mxl_sys::mxlFlowReader, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlReleaseFlowReader"] + mxl_release_flow_reader: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + reader: mxl_sys::mxlFlowReader, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlCreateFlowWriter"] + mxl_create_flow_writer: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + flowId: *const std::os::raw::c_char, + options: *const std::os::raw::c_char, + writer: *mut mxl_sys::mxlFlowWriter, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlReleaseFlowWriter"] + mxl_release_flow_writer: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + writer: mxl_sys::mxlFlowWriter, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowReaderGetInfo"] + mxl_flow_reader_get_info: unsafe extern "C" fn( + reader: mxl_sys::mxlFlowReader, + info: *mut mxl_sys::FlowInfo, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowReaderGetGrain"] + mxl_flow_reader_get_grain: unsafe extern "C" fn( + reader: mxl_sys::mxlFlowReader, + index: u64, + timeoutNs: u64, + grain: *mut mxl_sys::GrainInfo, + payload: *mut *mut u8, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowReaderGetGrainNonBlocking"] + mxl_flow_reader_get_grain_non_blocking: unsafe extern "C" fn( + reader: mxl_sys::mxlFlowReader, + index: u64, + grain: *mut mxl_sys::GrainInfo, + payload: *mut *mut u8, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowWriterOpenGrain"] + mxl_flow_writer_open_grain: unsafe extern "C" fn( + writer: mxl_sys::mxlFlowWriter, + index: u64, + grainInfo: *mut mxl_sys::GrainInfo, + payload: *mut *mut u8, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCancelGrain"] + mxl_flow_writer_cancel_grain: + unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCommitGrain"] + mxl_flow_writer_commit_grain: unsafe extern "C" fn( + writer: mxl_sys::mxlFlowWriter, + grain: *const mxl_sys::GrainInfo, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowReaderGetSamples"] + mxl_flow_reader_get_samples: unsafe extern "C" fn( + reader: mxl_sys::mxlFlowReader, + index: u64, + count: usize, + payloadBuffersSlices: *mut mxl_sys::WrappedMultiBufferSlice, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowWriterOpenSamples"] + mxl_flow_writer_open_samples: unsafe extern "C" fn( + writer: mxl_sys::mxlFlowWriter, + index: u64, + count: usize, + payloadBuffersSlices: *mut mxl_sys::MutableWrappedMultiBufferSlice, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCancelSamples"] + mxl_flow_writer_cancel_samples: + unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCommitSamples"] + mxl_flow_writer_commit_samples: + unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlGetCurrentHeadIndex"] + mxl_get_current_head_index: unsafe extern "C" fn(editRate: *const mxl_sys::Rational) -> u64, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlGetNsUntilHeadIndex"] + mxl_get_ns_until_head_index: + unsafe extern "C" fn(index: u64, editRate: *const mxl_sys::Rational) -> u64, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlTimestampToHeadIndex"] + mxl_timestamp_to_head_index: + unsafe extern "C" fn(editRate: *const mxl_sys::Rational, timestamp: u64) -> u64, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlHeadIndexToTimestamp"] + mxl_head_index_to_timestamp: + unsafe extern "C" fn(editRate: *const mxl_sys::Rational, index: u64) -> u64, + #[dlopen2_name = "mxlSleepForNs"] + mxl_sleep_for_ns: unsafe extern "C" fn(ns: u64), + #[dlopen2_name = "mxlGetTime"] + mxl_get_time: unsafe extern "C" fn() -> u64, +} + +pub fn load_api(path_to_so_file: impl AsRef) -> Result, dlopen2::Error> { + unsafe { Container::load(path_to_so_file.as_ref().as_os_str()) } +} + +pub struct MxlInstance { + api: Container, + instance: mxl_sys::mxlInstance, +} + +impl MxlInstance { + pub fn new(api: Container, domain: &str, options: &str) -> Result { + let instance = unsafe { + api.mxl_create_instance( + CString::new(domain).unwrap().as_ptr(), + CString::new(options).unwrap().as_ptr(), + ) + }; + if instance.is_null() { + Err(MxlError::Other( + "Failed to create MXL instance.".to_string(), + )) + } else { + Ok(Self { api, instance }) + } + } + + pub fn destroy(&mut self) -> Result<(), MxlError> { + let result; + if self.instance.is_null() { + return Err(MxlError::Other( + "Internal instance not initialized.".to_string(), + )); + } + unsafe { + result = status_to_result(self.api.mxl_destroy_instance(self.instance)); + } + self.instance = std::ptr::null_mut(); + result + } + + pub fn create_flow_reader(&self, flow_id: &str) -> Result { + let flow_id = CString::new(flow_id).map_err(|_| MxlError::InvalidArg)?; + let options = CString::new("").map_err(|_| MxlError::InvalidArg)?; + let mut reader: mxl_sys::mxlFlowReader = std::ptr::null_mut(); + unsafe { + status_to_result(self.api.mxl_create_flow_reader( + self.instance, + flow_id.as_ptr(), + options.as_ptr(), + &mut reader, + ))?; + } + if reader.is_null() { + return Err(MxlError::Other("Failed to create flow reader.".to_string())); + } + Ok(MxlFlowReader { + instance: self, + reader, + }) + } + + pub fn get_current_head_index(&self, rational: &mxl_sys::Rational) -> u64 { + unsafe { self.api.mxl_get_current_head_index(rational) } + } +} + +impl Drop for MxlInstance { + fn drop(&mut self) { + if !self.instance.is_null() { + if let Err(error) = self.destroy() { + tracing::error!("Failed to destroy MXL instance: {:?}", error); + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DataFormat { + Unspecified, + Video, + Audio, + Data, + Mux, +} + +impl From for DataFormat { + fn from(value: u32) -> Self { + match value { + 0 => DataFormat::Unspecified, + 1 => DataFormat::Video, + 2 => DataFormat::Audio, + 3 => DataFormat::Data, + 4 => DataFormat::Mux, + _ => DataFormat::Unspecified, + } + } +} + +pub struct FlowInfo { + value: mxl_sys::FlowInfo, +} + +impl FlowInfo { + pub fn discrete_flow_info(&self) -> Result<&mxl_sys::DiscreteFlowInfo, MxlError> { + // Check is based on mxlIsDiscreteDataFormat, which is inline, thus not accessible in + // mxl_sys. + if self.value.common.format != mxl_sys::MXL_DATA_FORMAT_VIDEO + && self.value.common.format != mxl_sys::MXL_DATA_FORMAT_DATA + { + return Err(MxlError::Other(format!( + "Flow format is {}, video or data require.", + self.value.common.format + ))); + } + Ok(unsafe { &self.value.__bindgen_anon_1.discrete }) + } +} + +pub struct GrainData { + pub user_data: Vec, + pub payload: Vec, +} + +pub struct MxlFlowReader<'a> { + instance: &'a MxlInstance, + reader: mxl_sys::mxlFlowReader, +} + +impl<'a> MxlFlowReader<'a> { + pub fn destroy(&mut self) -> Result<(), MxlError> { + let result; + if self.reader.is_null() { + return Err(MxlError::InvalidArg); + } + unsafe { + result = status_to_result( + self.instance + .api + .mxl_release_flow_reader(self.instance.instance, self.reader), + ); + } + self.reader = std::ptr::null_mut(); + result + } + + pub fn get_info(&self) -> Result { + let mut flow_info: mxl_sys::FlowInfo = unsafe { std::mem::zeroed() }; + unsafe { + status_to_result( + self.instance + .api + .mxl_flow_reader_get_info(self.reader, &mut flow_info), + )?; + } + Ok(FlowInfo { value: flow_info }) + } + + pub fn get_complete_grain(&self, index: u64, timeout: Duration) -> Result { + let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; + let mut payload_ptr: *mut u8 = std::ptr::null_mut(); + let timeout_ns = timeout.as_nanos() as u64; + loop { + unsafe { + status_to_result(self.instance.api.mxl_flow_reader_get_grain( + self.reader, + index, + timeout_ns, + &mut grain_info, + &mut payload_ptr, + ))?; + } + if grain_info.commitedSize != grain_info.grainSize { + // We don't need partial grains. Wait for the grain to be complete. + continue; + } + if payload_ptr.is_null() { + return Err(MxlError::Other(format!( + "Failed to get grain payload for index {}.", + index + ))); + } + break; + } + let payload = unsafe { + let slice = std::slice::from_raw_parts(payload_ptr, grain_info.grainSize as usize); + slice.to_vec() + }; + let user_data = grain_info.userData.to_vec(); + Ok(GrainData { user_data, payload }) + } +} + +impl Drop for MxlFlowReader<'_> { + fn drop(&mut self) { + if !self.reader.is_null() { + if let Err(error) = self.destroy() { + tracing::error!("Failed to release MXL flow reader: {:?}", error); + } + } + } +} diff --git a/rust-bindings/mxl/tests/basic_tests.rs b/rust-bindings/mxl/tests/basic_tests.rs new file mode 100644 index 000000000..bdd6f69cf --- /dev/null +++ b/rust-bindings/mxl/tests/basic_tests.rs @@ -0,0 +1,54 @@ +/// Tests of the basic low level synchronous API. +/// +/// The tests now require an MXL library of a specific name to be present in the system. This should +/// change in the future. For now, feel free to just edit the path to your library. +use std::path::PathBuf; +use std::time::Duration; + +use tracing::info; + +static LOG_ONCE: std::sync::Once = std::sync::Once::new(); + +fn setup_test() -> mxl::MxlInstance { + // Set up the logging to use the RUST_LOG environment variable and if not present, print INFO + // and higher. + LOG_ONCE.call_once(|| { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::builder() + .with_default_directive(tracing::level_filters::LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .init(); + }); + + // To run the test, you need to alter the path below to point to a valid file with MXL + // implementation. It is possible to use just the file name, as long as the file is in the + // library search path. + let libpath = + PathBuf::from("/home/pac/Sources/MXL/mxl/install/Linux-Clang-Debug/lib/libmxl.so"); + let mxl_api = mxl::load_api(libpath).unwrap(); + // TODO: Randomize the domain name to allow parallel tests run. + mxl::MxlInstance::new(mxl_api, "/dev/shm/mxl_domain", "").unwrap() +} + +#[test] +fn basic_mxl_writing_reading() { + // TODO: Add the writing part of the test. Now the test requires the external MXL tool to write + // data: `tools/mxl-gst/mxl-gst-videotestsrc -d /dev/shm/mxl_domain -f + // ../../lib/tests/data/v210_flow.json` + let mut mxl_instance = setup_test(); + let mut flow_reader = mxl_instance + .create_flow_reader("5fbec3b1-1b0f-417d-9059-8b94a47197ed") + .unwrap(); + let flow_info = flow_reader.get_info().unwrap(); + let rate = flow_info.discrete_flow_info().unwrap().grainRate; + let current_head_index = mxl_instance.get_current_head_index(&rate); + let grain_data = flow_reader + .get_complete_grain(current_head_index, Duration::from_secs(5)) + .unwrap(); + info!("Grain data len: {:?}", grain_data.payload.len()); + flow_reader.destroy().unwrap(); + drop(flow_reader); + mxl_instance.destroy().unwrap(); +} From 4ae3acc086a256661e1c48a169027bb1c6632ef4 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 15:16:47 +0000 Subject: [PATCH 03/77] vscode: set root for rust analyzer Signed-off-by: Pedro Ferreira --- .vscode/settings.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ae5743569..9ab20c015 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project https://github.com/dmf-mxl/mxl/contributors.md // SPDX-License-Identifier: Apache-2.0 { + "rust-analyzer.linkedProjects": [ + "rust-bindings/Cargo.toml" + ], "files.associations": { "array": "cpp", "atomic": "cpp", @@ -71,4 +74,4 @@ "typeinfo": "cpp", "variant": "cpp" } -} \ No newline at end of file +} From fbb907d7ea5ed05331df29325166e3a7bf5f56ef Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 10:29:59 +0000 Subject: [PATCH 04/77] modules: create a separate module for errors Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/src/error.rs | 47 ++++++++++++++++++ rust-bindings/mxl/src/lib.rs | 91 ++++++++++------------------------ 2 files changed, 73 insertions(+), 65 deletions(-) create mode 100644 rust-bindings/mxl/src/error.rs diff --git a/rust-bindings/mxl/src/error.rs b/rust-bindings/mxl/src/error.rs new file mode 100644 index 000000000..4c1327bbe --- /dev/null +++ b/rust-bindings/mxl/src/error.rs @@ -0,0 +1,47 @@ +pub type Result = core::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Unknown error")] + Unknown, + #[error("Flow not found")] + FlowNotFound, + #[error("Out of range - too late")] + OutOfRangeTooLate, + #[error("Out of range - too early")] + OutOfRangeTooEarly, + #[error("Invalid flow reader")] + InvalidFlowReader, + #[error("Invalid flow writer")] + InvalidFlowWriter, + #[error("Timeout")] + Timeout, + #[error("Invalid argument")] + InvalidArg, + #[error("Conflict")] + Conflict, + /// The error is not defined in the MXL API, but it is used to wrap other errors. + #[error("Other error: {0}")] + Other(String), + + #[error("dlopen: {0}")] + DlOpen(#[from] dlopen2::Error), +} + +impl Error { + pub fn from_status(status: mxl_sys::mxlStatus) -> Result<()> { + match status { + mxl_sys::MXL_STATUS_OK => Ok(()), + mxl_sys::MXL_ERR_UNKNOWN => Err(Error::Unknown), + mxl_sys::MXL_ERR_FLOW_NOT_FOUND => Err(Error::FlowNotFound), + mxl_sys::MXL_ERR_OUT_OF_RANGE_TOO_LATE => Err(Error::OutOfRangeTooLate), + mxl_sys::MXL_ERR_OUT_OF_RANGE_TOO_EARLY => Err(Error::OutOfRangeTooEarly), + mxl_sys::MXL_ERR_INVALID_FLOW_READER => Err(Error::InvalidFlowReader), + mxl_sys::MXL_ERR_INVALID_FLOW_WRITER => Err(Error::InvalidFlowWriter), + mxl_sys::MXL_ERR_TIMEOUT => Err(Error::Timeout), + mxl_sys::MXL_ERR_INVALID_ARG => Err(Error::InvalidArg), + mxl_sys::MXL_ERR_CONFLICT => Err(Error::Conflict), + _ => Err(Error::Unknown), + } + } +} diff --git a/rust-bindings/mxl/src/lib.rs b/rust-bindings/mxl/src/lib.rs index ada237b28..07b62fc4c 100644 --- a/rust-bindings/mxl/src/lib.rs +++ b/rust-bindings/mxl/src/lib.rs @@ -1,50 +1,13 @@ +mod error; + +pub use error::{Error, Result}; + use std::ffi::CString; use std::path::Path; use std::time::Duration; use dlopen2::wrapper::{Container, WrapperApi}; -#[derive(Debug, thiserror::Error)] -pub enum MxlError { - #[error("Unknown error")] - Unknown, - #[error("Flow not found")] - FlowNotFound, - #[error("Out of range - too late")] - OutOfRangeTooLate, - #[error("Out of range - too early")] - OutOfRangeTooEarly, - #[error("Invalid flow reader")] - InvalidFlowReader, - #[error("Invalid flow writer")] - InvalidFlowWriter, - #[error("Timeout")] - Timeout, - #[error("Invalid argument")] - InvalidArg, - #[error("Conflict")] - Conflict, - /// The error is not defined in the MXL API, but it is used to wrap other errors. - #[error("Other error: {0}")] - Other(String), -} - -fn status_to_result(status: mxl_sys::mxlStatus) -> Result<(), MxlError> { - match status { - mxl_sys::MXL_STATUS_OK => Ok(()), - mxl_sys::MXL_ERR_UNKNOWN => Err(MxlError::Unknown), - mxl_sys::MXL_ERR_FLOW_NOT_FOUND => Err(MxlError::FlowNotFound), - mxl_sys::MXL_ERR_OUT_OF_RANGE_TOO_LATE => Err(MxlError::OutOfRangeTooLate), - mxl_sys::MXL_ERR_OUT_OF_RANGE_TOO_EARLY => Err(MxlError::OutOfRangeTooEarly), - mxl_sys::MXL_ERR_INVALID_FLOW_READER => Err(MxlError::InvalidFlowReader), - mxl_sys::MXL_ERR_INVALID_FLOW_WRITER => Err(MxlError::InvalidFlowWriter), - mxl_sys::MXL_ERR_TIMEOUT => Err(MxlError::Timeout), - mxl_sys::MXL_ERR_INVALID_ARG => Err(MxlError::InvalidArg), - mxl_sys::MXL_ERR_CONFLICT => Err(MxlError::Conflict), - _ => Err(MxlError::Unknown), - } -} - #[derive(WrapperApi)] pub struct MxlApi { #[dlopen2_name = "mxlGetVersion"] @@ -182,8 +145,8 @@ pub struct MxlApi { mxl_get_time: unsafe extern "C" fn() -> u64, } -pub fn load_api(path_to_so_file: impl AsRef) -> Result, dlopen2::Error> { - unsafe { Container::load(path_to_so_file.as_ref().as_os_str()) } +pub fn load_api(path_to_so_file: impl AsRef) -> Result> { + Ok(unsafe { Container::load(path_to_so_file.as_ref().as_os_str()) }?) } pub struct MxlInstance { @@ -192,7 +155,7 @@ pub struct MxlInstance { } impl MxlInstance { - pub fn new(api: Container, domain: &str, options: &str) -> Result { + pub fn new(api: Container, domain: &str, options: &str) -> Result { let instance = unsafe { api.mxl_create_instance( CString::new(domain).unwrap().as_ptr(), @@ -200,34 +163,32 @@ impl MxlInstance { ) }; if instance.is_null() { - Err(MxlError::Other( - "Failed to create MXL instance.".to_string(), - )) + Err(Error::Other("Failed to create MXL instance.".to_string())) } else { Ok(Self { api, instance }) } } - pub fn destroy(&mut self) -> Result<(), MxlError> { + pub fn destroy(&mut self) -> Result<()> { let result; if self.instance.is_null() { - return Err(MxlError::Other( + return Err(Error::Other( "Internal instance not initialized.".to_string(), )); } unsafe { - result = status_to_result(self.api.mxl_destroy_instance(self.instance)); + result = Error::from_status(self.api.mxl_destroy_instance(self.instance)); } self.instance = std::ptr::null_mut(); result } - pub fn create_flow_reader(&self, flow_id: &str) -> Result { - let flow_id = CString::new(flow_id).map_err(|_| MxlError::InvalidArg)?; - let options = CString::new("").map_err(|_| MxlError::InvalidArg)?; + pub fn create_flow_reader(&self, flow_id: &str) -> Result { + let flow_id = CString::new(flow_id).map_err(|_| Error::InvalidArg)?; + let options = CString::new("").map_err(|_| Error::InvalidArg)?; let mut reader: mxl_sys::mxlFlowReader = std::ptr::null_mut(); unsafe { - status_to_result(self.api.mxl_create_flow_reader( + Error::from_status(self.api.mxl_create_flow_reader( self.instance, flow_id.as_ptr(), options.as_ptr(), @@ -235,7 +196,7 @@ impl MxlInstance { ))?; } if reader.is_null() { - return Err(MxlError::Other("Failed to create flow reader.".to_string())); + return Err(Error::Other("Failed to create flow reader.".to_string())); } Ok(MxlFlowReader { instance: self, @@ -285,13 +246,13 @@ pub struct FlowInfo { } impl FlowInfo { - pub fn discrete_flow_info(&self) -> Result<&mxl_sys::DiscreteFlowInfo, MxlError> { + pub fn discrete_flow_info(&self) -> Result<&mxl_sys::DiscreteFlowInfo> { // Check is based on mxlIsDiscreteDataFormat, which is inline, thus not accessible in // mxl_sys. if self.value.common.format != mxl_sys::MXL_DATA_FORMAT_VIDEO && self.value.common.format != mxl_sys::MXL_DATA_FORMAT_DATA { - return Err(MxlError::Other(format!( + return Err(Error::Other(format!( "Flow format is {}, video or data require.", self.value.common.format ))); @@ -311,13 +272,13 @@ pub struct MxlFlowReader<'a> { } impl<'a> MxlFlowReader<'a> { - pub fn destroy(&mut self) -> Result<(), MxlError> { + pub fn destroy(&mut self) -> Result<()> { let result; if self.reader.is_null() { - return Err(MxlError::InvalidArg); + return Err(Error::InvalidArg); } unsafe { - result = status_to_result( + result = Error::from_status( self.instance .api .mxl_release_flow_reader(self.instance.instance, self.reader), @@ -327,10 +288,10 @@ impl<'a> MxlFlowReader<'a> { result } - pub fn get_info(&self) -> Result { + pub fn get_info(&self) -> Result { let mut flow_info: mxl_sys::FlowInfo = unsafe { std::mem::zeroed() }; unsafe { - status_to_result( + Error::from_status( self.instance .api .mxl_flow_reader_get_info(self.reader, &mut flow_info), @@ -339,13 +300,13 @@ impl<'a> MxlFlowReader<'a> { Ok(FlowInfo { value: flow_info }) } - pub fn get_complete_grain(&self, index: u64, timeout: Duration) -> Result { + pub fn get_complete_grain(&self, index: u64, timeout: Duration) -> Result { let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; let mut payload_ptr: *mut u8 = std::ptr::null_mut(); let timeout_ns = timeout.as_nanos() as u64; loop { unsafe { - status_to_result(self.instance.api.mxl_flow_reader_get_grain( + Error::from_status(self.instance.api.mxl_flow_reader_get_grain( self.reader, index, timeout_ns, @@ -358,7 +319,7 @@ impl<'a> MxlFlowReader<'a> { continue; } if payload_ptr.is_null() { - return Err(MxlError::Other(format!( + return Err(Error::Other(format!( "Failed to get grain payload for index {}.", index ))); From e595c2070f3ea92dc1527b8e0b5eb6ddc54869cc Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 10:34:59 +0000 Subject: [PATCH 05/77] modules: create a separate module for the ffi api Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/src/api.rs | 146 +++++++++++++++++++++++++++++++++++ rust-bindings/mxl/src/lib.rs | 146 +---------------------------------- 2 files changed, 149 insertions(+), 143 deletions(-) create mode 100644 rust-bindings/mxl/src/api.rs diff --git a/rust-bindings/mxl/src/api.rs b/rust-bindings/mxl/src/api.rs new file mode 100644 index 000000000..f4941ba30 --- /dev/null +++ b/rust-bindings/mxl/src/api.rs @@ -0,0 +1,146 @@ +use std::path::Path; + +use dlopen2::wrapper::{Container, WrapperApi}; + +use crate::Result; + +#[derive(WrapperApi)] +pub struct MxlApi { + #[dlopen2_name = "mxlGetVersion"] + mxl_get_version: + unsafe extern "C" fn(out_version: *mut mxl_sys::mxlVersionType) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlCreateInstance"] + mxl_create_instance: unsafe extern "C" fn( + in_mxlDomain: *const std::os::raw::c_char, + in_options: *const std::os::raw::c_char, + ) -> mxl_sys::mxlInstance, + #[dlopen2_name = "mxlGarbageCollectFlows"] + mxl_garbage_collect_flows: + unsafe extern "C" fn(in_instance: mxl_sys::mxlInstance) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlDestroyInstance"] + mxl_destroy_instance: + unsafe extern "C" fn(in_instance: mxl_sys::mxlInstance) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlCreateFlow"] + mxl_create_flow: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + flowDef: *const std::os::raw::c_char, + options: *const std::os::raw::c_char, + info: *mut mxl_sys::FlowInfo, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlDestroyFlow"] + mxl_destroy_flow: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + flowId: *const std::os::raw::c_char, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlCreateFlowReader"] + mxl_create_flow_reader: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + flowId: *const std::os::raw::c_char, + options: *const std::os::raw::c_char, + reader: *mut mxl_sys::mxlFlowReader, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlReleaseFlowReader"] + mxl_release_flow_reader: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + reader: mxl_sys::mxlFlowReader, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlCreateFlowWriter"] + mxl_create_flow_writer: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + flowId: *const std::os::raw::c_char, + options: *const std::os::raw::c_char, + writer: *mut mxl_sys::mxlFlowWriter, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlReleaseFlowWriter"] + mxl_release_flow_writer: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + writer: mxl_sys::mxlFlowWriter, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowReaderGetInfo"] + mxl_flow_reader_get_info: unsafe extern "C" fn( + reader: mxl_sys::mxlFlowReader, + info: *mut mxl_sys::FlowInfo, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowReaderGetGrain"] + mxl_flow_reader_get_grain: unsafe extern "C" fn( + reader: mxl_sys::mxlFlowReader, + index: u64, + timeoutNs: u64, + grain: *mut mxl_sys::GrainInfo, + payload: *mut *mut u8, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowReaderGetGrainNonBlocking"] + mxl_flow_reader_get_grain_non_blocking: unsafe extern "C" fn( + reader: mxl_sys::mxlFlowReader, + index: u64, + grain: *mut mxl_sys::GrainInfo, + payload: *mut *mut u8, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowWriterOpenGrain"] + mxl_flow_writer_open_grain: unsafe extern "C" fn( + writer: mxl_sys::mxlFlowWriter, + index: u64, + grainInfo: *mut mxl_sys::GrainInfo, + payload: *mut *mut u8, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCancelGrain"] + mxl_flow_writer_cancel_grain: + unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCommitGrain"] + mxl_flow_writer_commit_grain: unsafe extern "C" fn( + writer: mxl_sys::mxlFlowWriter, + grain: *const mxl_sys::GrainInfo, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowReaderGetSamples"] + mxl_flow_reader_get_samples: unsafe extern "C" fn( + reader: mxl_sys::mxlFlowReader, + index: u64, + count: usize, + payloadBuffersSlices: *mut mxl_sys::WrappedMultiBufferSlice, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowWriterOpenSamples"] + mxl_flow_writer_open_samples: unsafe extern "C" fn( + writer: mxl_sys::mxlFlowWriter, + index: u64, + count: usize, + payloadBuffersSlices: *mut mxl_sys::MutableWrappedMultiBufferSlice, + ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCancelSamples"] + mxl_flow_writer_cancel_samples: + unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCommitSamples"] + mxl_flow_writer_commit_samples: + unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlGetCurrentHeadIndex"] + mxl_get_current_head_index: unsafe extern "C" fn(editRate: *const mxl_sys::Rational) -> u64, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlGetNsUntilHeadIndex"] + mxl_get_ns_until_head_index: + unsafe extern "C" fn(index: u64, editRate: *const mxl_sys::Rational) -> u64, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlTimestampToHeadIndex"] + mxl_timestamp_to_head_index: + unsafe extern "C" fn(editRate: *const mxl_sys::Rational, timestamp: u64) -> u64, + #[allow(non_snake_case)] + #[dlopen2_name = "mxlHeadIndexToTimestamp"] + mxl_head_index_to_timestamp: + unsafe extern "C" fn(editRate: *const mxl_sys::Rational, index: u64) -> u64, + #[dlopen2_name = "mxlSleepForNs"] + mxl_sleep_for_ns: unsafe extern "C" fn(ns: u64), + #[dlopen2_name = "mxlGetTime"] + mxl_get_time: unsafe extern "C" fn() -> u64, +} + +pub fn load_api(path_to_so_file: impl AsRef) -> Result> { + Ok(unsafe { Container::load(path_to_so_file.as_ref().as_os_str()) }?) +} diff --git a/rust-bindings/mxl/src/lib.rs b/rust-bindings/mxl/src/lib.rs index 07b62fc4c..61c6a363b 100644 --- a/rust-bindings/mxl/src/lib.rs +++ b/rust-bindings/mxl/src/lib.rs @@ -1,153 +1,13 @@ +mod api; mod error; +pub use api::{MxlApi, load_api}; pub use error::{Error, Result}; use std::ffi::CString; -use std::path::Path; use std::time::Duration; -use dlopen2::wrapper::{Container, WrapperApi}; - -#[derive(WrapperApi)] -pub struct MxlApi { - #[dlopen2_name = "mxlGetVersion"] - mxl_get_version: - unsafe extern "C" fn(out_version: *mut mxl_sys::mxlVersionType) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlCreateInstance"] - mxl_create_instance: unsafe extern "C" fn( - in_mxlDomain: *const std::os::raw::c_char, - in_options: *const std::os::raw::c_char, - ) -> mxl_sys::mxlInstance, - #[dlopen2_name = "mxlGarbageCollectFlows"] - mxl_garbage_collect_flows: - unsafe extern "C" fn(in_instance: mxl_sys::mxlInstance) -> mxl_sys::mxlStatus, - #[dlopen2_name = "mxlDestroyInstance"] - mxl_destroy_instance: - unsafe extern "C" fn(in_instance: mxl_sys::mxlInstance) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlCreateFlow"] - mxl_create_flow: unsafe extern "C" fn( - instance: mxl_sys::mxlInstance, - flowDef: *const std::os::raw::c_char, - options: *const std::os::raw::c_char, - info: *mut mxl_sys::FlowInfo, - ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlDestroyFlow"] - mxl_destroy_flow: unsafe extern "C" fn( - instance: mxl_sys::mxlInstance, - flowId: *const std::os::raw::c_char, - ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlCreateFlowReader"] - mxl_create_flow_reader: unsafe extern "C" fn( - instance: mxl_sys::mxlInstance, - flowId: *const std::os::raw::c_char, - options: *const std::os::raw::c_char, - reader: *mut mxl_sys::mxlFlowReader, - ) -> mxl_sys::mxlStatus, - #[dlopen2_name = "mxlReleaseFlowReader"] - mxl_release_flow_reader: unsafe extern "C" fn( - instance: mxl_sys::mxlInstance, - reader: mxl_sys::mxlFlowReader, - ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlCreateFlowWriter"] - mxl_create_flow_writer: unsafe extern "C" fn( - instance: mxl_sys::mxlInstance, - flowId: *const std::os::raw::c_char, - options: *const std::os::raw::c_char, - writer: *mut mxl_sys::mxlFlowWriter, - ) -> mxl_sys::mxlStatus, - #[dlopen2_name = "mxlReleaseFlowWriter"] - mxl_release_flow_writer: unsafe extern "C" fn( - instance: mxl_sys::mxlInstance, - writer: mxl_sys::mxlFlowWriter, - ) -> mxl_sys::mxlStatus, - #[dlopen2_name = "mxlFlowReaderGetInfo"] - mxl_flow_reader_get_info: unsafe extern "C" fn( - reader: mxl_sys::mxlFlowReader, - info: *mut mxl_sys::FlowInfo, - ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlFlowReaderGetGrain"] - mxl_flow_reader_get_grain: unsafe extern "C" fn( - reader: mxl_sys::mxlFlowReader, - index: u64, - timeoutNs: u64, - grain: *mut mxl_sys::GrainInfo, - payload: *mut *mut u8, - ) -> mxl_sys::mxlStatus, - #[dlopen2_name = "mxlFlowReaderGetGrainNonBlocking"] - mxl_flow_reader_get_grain_non_blocking: unsafe extern "C" fn( - reader: mxl_sys::mxlFlowReader, - index: u64, - grain: *mut mxl_sys::GrainInfo, - payload: *mut *mut u8, - ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlFlowWriterOpenGrain"] - mxl_flow_writer_open_grain: unsafe extern "C" fn( - writer: mxl_sys::mxlFlowWriter, - index: u64, - grainInfo: *mut mxl_sys::GrainInfo, - payload: *mut *mut u8, - ) -> mxl_sys::mxlStatus, - #[dlopen2_name = "mxlFlowWriterCancelGrain"] - mxl_flow_writer_cancel_grain: - unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, - #[dlopen2_name = "mxlFlowWriterCommitGrain"] - mxl_flow_writer_commit_grain: unsafe extern "C" fn( - writer: mxl_sys::mxlFlowWriter, - grain: *const mxl_sys::GrainInfo, - ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlFlowReaderGetSamples"] - mxl_flow_reader_get_samples: unsafe extern "C" fn( - reader: mxl_sys::mxlFlowReader, - index: u64, - count: usize, - payloadBuffersSlices: *mut mxl_sys::WrappedMultiBufferSlice, - ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlFlowWriterOpenSamples"] - mxl_flow_writer_open_samples: unsafe extern "C" fn( - writer: mxl_sys::mxlFlowWriter, - index: u64, - count: usize, - payloadBuffersSlices: *mut mxl_sys::MutableWrappedMultiBufferSlice, - ) -> mxl_sys::mxlStatus, - #[dlopen2_name = "mxlFlowWriterCancelSamples"] - mxl_flow_writer_cancel_samples: - unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, - #[dlopen2_name = "mxlFlowWriterCommitSamples"] - mxl_flow_writer_commit_samples: - unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlGetCurrentHeadIndex"] - mxl_get_current_head_index: unsafe extern "C" fn(editRate: *const mxl_sys::Rational) -> u64, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlGetNsUntilHeadIndex"] - mxl_get_ns_until_head_index: - unsafe extern "C" fn(index: u64, editRate: *const mxl_sys::Rational) -> u64, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlTimestampToHeadIndex"] - mxl_timestamp_to_head_index: - unsafe extern "C" fn(editRate: *const mxl_sys::Rational, timestamp: u64) -> u64, - #[allow(non_snake_case)] - #[dlopen2_name = "mxlHeadIndexToTimestamp"] - mxl_head_index_to_timestamp: - unsafe extern "C" fn(editRate: *const mxl_sys::Rational, index: u64) -> u64, - #[dlopen2_name = "mxlSleepForNs"] - mxl_sleep_for_ns: unsafe extern "C" fn(ns: u64), - #[dlopen2_name = "mxlGetTime"] - mxl_get_time: unsafe extern "C" fn() -> u64, -} - -pub fn load_api(path_to_so_file: impl AsRef) -> Result> { - Ok(unsafe { Container::load(path_to_so_file.as_ref().as_os_str()) }?) -} +use dlopen2::wrapper::Container; pub struct MxlInstance { api: Container, From 09578cc51ae7135ab11a191a6ef9da00581c1cfa Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 10:38:22 +0000 Subject: [PATCH 06/77] modules: create a separate module for the instance Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/src/instance.rs | 75 +++++++++++++++++++++++++++++++ rust-bindings/mxl/src/lib.rs | 75 +------------------------------ 2 files changed, 77 insertions(+), 73 deletions(-) create mode 100644 rust-bindings/mxl/src/instance.rs diff --git a/rust-bindings/mxl/src/instance.rs b/rust-bindings/mxl/src/instance.rs new file mode 100644 index 000000000..f44704ce2 --- /dev/null +++ b/rust-bindings/mxl/src/instance.rs @@ -0,0 +1,75 @@ +use std::ffi::CString; + +use dlopen2::wrapper::Container; + +use crate::{Error, MxlApi, MxlFlowReader, Result}; + +pub struct MxlInstance { + pub(crate) api: Container, + pub(crate) instance: mxl_sys::mxlInstance, +} + +impl MxlInstance { + pub fn new(api: Container, domain: &str, options: &str) -> Result { + let instance = unsafe { + api.mxl_create_instance( + CString::new(domain).unwrap().as_ptr(), + CString::new(options).unwrap().as_ptr(), + ) + }; + if instance.is_null() { + Err(Error::Other("Failed to create MXL instance.".to_string())) + } else { + Ok(Self { api, instance }) + } + } + + pub fn destroy(&mut self) -> Result<()> { + let result; + if self.instance.is_null() { + return Err(Error::Other( + "Internal instance not initialized.".to_string(), + )); + } + unsafe { + result = Error::from_status(self.api.mxl_destroy_instance(self.instance)); + } + self.instance = std::ptr::null_mut(); + result + } + + pub fn create_flow_reader(&self, flow_id: &str) -> Result { + let flow_id = CString::new(flow_id).map_err(|_| Error::InvalidArg)?; + let options = CString::new("").map_err(|_| Error::InvalidArg)?; + let mut reader: mxl_sys::mxlFlowReader = std::ptr::null_mut(); + unsafe { + Error::from_status(self.api.mxl_create_flow_reader( + self.instance, + flow_id.as_ptr(), + options.as_ptr(), + &mut reader, + ))?; + } + if reader.is_null() { + return Err(Error::Other("Failed to create flow reader.".to_string())); + } + Ok(MxlFlowReader { + instance: self, + reader, + }) + } + + pub fn get_current_head_index(&self, rational: &mxl_sys::Rational) -> u64 { + unsafe { self.api.mxl_get_current_head_index(rational) } + } +} + +impl Drop for MxlInstance { + fn drop(&mut self) { + if !self.instance.is_null() { + if let Err(error) = self.destroy() { + tracing::error!("Failed to destroy MXL instance: {:?}", error); + } + } + } +} diff --git a/rust-bindings/mxl/src/lib.rs b/rust-bindings/mxl/src/lib.rs index 61c6a363b..4e6f735ad 100644 --- a/rust-bindings/mxl/src/lib.rs +++ b/rust-bindings/mxl/src/lib.rs @@ -1,84 +1,13 @@ mod api; mod error; +mod instance; pub use api::{MxlApi, load_api}; pub use error::{Error, Result}; +pub use instance::MxlInstance; -use std::ffi::CString; use std::time::Duration; -use dlopen2::wrapper::Container; - -pub struct MxlInstance { - api: Container, - instance: mxl_sys::mxlInstance, -} - -impl MxlInstance { - pub fn new(api: Container, domain: &str, options: &str) -> Result { - let instance = unsafe { - api.mxl_create_instance( - CString::new(domain).unwrap().as_ptr(), - CString::new(options).unwrap().as_ptr(), - ) - }; - if instance.is_null() { - Err(Error::Other("Failed to create MXL instance.".to_string())) - } else { - Ok(Self { api, instance }) - } - } - - pub fn destroy(&mut self) -> Result<()> { - let result; - if self.instance.is_null() { - return Err(Error::Other( - "Internal instance not initialized.".to_string(), - )); - } - unsafe { - result = Error::from_status(self.api.mxl_destroy_instance(self.instance)); - } - self.instance = std::ptr::null_mut(); - result - } - - pub fn create_flow_reader(&self, flow_id: &str) -> Result { - let flow_id = CString::new(flow_id).map_err(|_| Error::InvalidArg)?; - let options = CString::new("").map_err(|_| Error::InvalidArg)?; - let mut reader: mxl_sys::mxlFlowReader = std::ptr::null_mut(); - unsafe { - Error::from_status(self.api.mxl_create_flow_reader( - self.instance, - flow_id.as_ptr(), - options.as_ptr(), - &mut reader, - ))?; - } - if reader.is_null() { - return Err(Error::Other("Failed to create flow reader.".to_string())); - } - Ok(MxlFlowReader { - instance: self, - reader, - }) - } - - pub fn get_current_head_index(&self, rational: &mxl_sys::Rational) -> u64 { - unsafe { self.api.mxl_get_current_head_index(rational) } - } -} - -impl Drop for MxlInstance { - fn drop(&mut self) { - if !self.instance.is_null() { - if let Err(error) = self.destroy() { - tracing::error!("Failed to destroy MXL instance: {:?}", error); - } - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DataFormat { Unspecified, From 56893ad36c6ac5212da5cf394c559956c55342ec Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 10:40:39 +0000 Subject: [PATCH 07/77] modules: create a separate module for flow information Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/src/flow.rs | 48 ++++++++++++++++++++++++++++++++++ rust-bindings/mxl/src/lib.rs | 49 +++-------------------------------- 2 files changed, 51 insertions(+), 46 deletions(-) create mode 100644 rust-bindings/mxl/src/flow.rs diff --git a/rust-bindings/mxl/src/flow.rs b/rust-bindings/mxl/src/flow.rs new file mode 100644 index 000000000..6172a764a --- /dev/null +++ b/rust-bindings/mxl/src/flow.rs @@ -0,0 +1,48 @@ +use crate::{Error, Result}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DataFormat { + Unspecified, + Video, + Audio, + Data, + Mux, +} + +impl From for DataFormat { + fn from(value: u32) -> Self { + match value { + 0 => DataFormat::Unspecified, + 1 => DataFormat::Video, + 2 => DataFormat::Audio, + 3 => DataFormat::Data, + 4 => DataFormat::Mux, + _ => DataFormat::Unspecified, + } + } +} + +pub struct FlowInfo { + pub(crate) value: mxl_sys::FlowInfo, +} + +impl FlowInfo { + pub fn discrete_flow_info(&self) -> Result<&mxl_sys::DiscreteFlowInfo> { + // Check is based on mxlIsDiscreteDataFormat, which is inline, thus not accessible in + // mxl_sys. + if self.value.common.format != mxl_sys::MXL_DATA_FORMAT_VIDEO + && self.value.common.format != mxl_sys::MXL_DATA_FORMAT_DATA + { + return Err(Error::Other(format!( + "Flow format is {}, video or data require.", + self.value.common.format + ))); + } + Ok(unsafe { &self.value.__bindgen_anon_1.discrete }) + } +} + +pub struct GrainData { + pub user_data: Vec, + pub payload: Vec, +} diff --git a/rust-bindings/mxl/src/lib.rs b/rust-bindings/mxl/src/lib.rs index 4e6f735ad..c15897f46 100644 --- a/rust-bindings/mxl/src/lib.rs +++ b/rust-bindings/mxl/src/lib.rs @@ -1,59 +1,16 @@ mod api; mod error; +mod flow; mod instance; pub use api::{MxlApi, load_api}; pub use error::{Error, Result}; +pub use flow::*; pub use instance::MxlInstance; use std::time::Duration; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum DataFormat { - Unspecified, - Video, - Audio, - Data, - Mux, -} - -impl From for DataFormat { - fn from(value: u32) -> Self { - match value { - 0 => DataFormat::Unspecified, - 1 => DataFormat::Video, - 2 => DataFormat::Audio, - 3 => DataFormat::Data, - 4 => DataFormat::Mux, - _ => DataFormat::Unspecified, - } - } -} - -pub struct FlowInfo { - value: mxl_sys::FlowInfo, -} - -impl FlowInfo { - pub fn discrete_flow_info(&self) -> Result<&mxl_sys::DiscreteFlowInfo> { - // Check is based on mxlIsDiscreteDataFormat, which is inline, thus not accessible in - // mxl_sys. - if self.value.common.format != mxl_sys::MXL_DATA_FORMAT_VIDEO - && self.value.common.format != mxl_sys::MXL_DATA_FORMAT_DATA - { - return Err(Error::Other(format!( - "Flow format is {}, video or data require.", - self.value.common.format - ))); - } - Ok(unsafe { &self.value.__bindgen_anon_1.discrete }) - } -} - -pub struct GrainData { - pub user_data: Vec, - pub payload: Vec, -} +use crate::flow::{FlowInfo, GrainData}; pub struct MxlFlowReader<'a> { instance: &'a MxlInstance, From 4f3d529e3fe9a1f827a10c6fca57616cf0d04253 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 10:43:52 +0000 Subject: [PATCH 08/77] modules: create a separate module for the flow reader Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/src/flow_reader.rs | 85 ++++++++++++++++++++++++++++ rust-bindings/mxl/src/lib.rs | 85 +--------------------------- 2 files changed, 87 insertions(+), 83 deletions(-) create mode 100644 rust-bindings/mxl/src/flow_reader.rs diff --git a/rust-bindings/mxl/src/flow_reader.rs b/rust-bindings/mxl/src/flow_reader.rs new file mode 100644 index 000000000..69b9a7805 --- /dev/null +++ b/rust-bindings/mxl/src/flow_reader.rs @@ -0,0 +1,85 @@ +use std::time::Duration; + +use crate::{ + Error, MxlInstance, Result, + flow::{FlowInfo, GrainData}, +}; + +pub struct MxlFlowReader<'a> { + pub(crate) instance: &'a MxlInstance, + pub(crate) reader: mxl_sys::mxlFlowReader, +} + +impl<'a> MxlFlowReader<'a> { + pub fn destroy(&mut self) -> Result<()> { + let result; + if self.reader.is_null() { + return Err(Error::InvalidArg); + } + unsafe { + result = Error::from_status( + self.instance + .api + .mxl_release_flow_reader(self.instance.instance, self.reader), + ); + } + self.reader = std::ptr::null_mut(); + result + } + + pub fn get_info(&self) -> Result { + let mut flow_info: mxl_sys::FlowInfo = unsafe { std::mem::zeroed() }; + unsafe { + Error::from_status( + self.instance + .api + .mxl_flow_reader_get_info(self.reader, &mut flow_info), + )?; + } + Ok(FlowInfo { value: flow_info }) + } + + pub fn get_complete_grain(&self, index: u64, timeout: Duration) -> Result { + let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; + let mut payload_ptr: *mut u8 = std::ptr::null_mut(); + let timeout_ns = timeout.as_nanos() as u64; + loop { + unsafe { + Error::from_status(self.instance.api.mxl_flow_reader_get_grain( + self.reader, + index, + timeout_ns, + &mut grain_info, + &mut payload_ptr, + ))?; + } + if grain_info.commitedSize != grain_info.grainSize { + // We don't need partial grains. Wait for the grain to be complete. + continue; + } + if payload_ptr.is_null() { + return Err(Error::Other(format!( + "Failed to get grain payload for index {}.", + index + ))); + } + break; + } + let payload = unsafe { + let slice = std::slice::from_raw_parts(payload_ptr, grain_info.grainSize as usize); + slice.to_vec() + }; + let user_data = grain_info.userData.to_vec(); + Ok(GrainData { user_data, payload }) + } +} + +impl Drop for MxlFlowReader<'_> { + fn drop(&mut self) { + if !self.reader.is_null() { + if let Err(error) = self.destroy() { + tracing::error!("Failed to release MXL flow reader: {:?}", error); + } + } + } +} diff --git a/rust-bindings/mxl/src/lib.rs b/rust-bindings/mxl/src/lib.rs index c15897f46..d9e679ae8 100644 --- a/rust-bindings/mxl/src/lib.rs +++ b/rust-bindings/mxl/src/lib.rs @@ -1,92 +1,11 @@ mod api; mod error; mod flow; +mod flow_reader; mod instance; pub use api::{MxlApi, load_api}; pub use error::{Error, Result}; pub use flow::*; +pub use flow_reader::MxlFlowReader; pub use instance::MxlInstance; - -use std::time::Duration; - -use crate::flow::{FlowInfo, GrainData}; - -pub struct MxlFlowReader<'a> { - instance: &'a MxlInstance, - reader: mxl_sys::mxlFlowReader, -} - -impl<'a> MxlFlowReader<'a> { - pub fn destroy(&mut self) -> Result<()> { - let result; - if self.reader.is_null() { - return Err(Error::InvalidArg); - } - unsafe { - result = Error::from_status( - self.instance - .api - .mxl_release_flow_reader(self.instance.instance, self.reader), - ); - } - self.reader = std::ptr::null_mut(); - result - } - - pub fn get_info(&self) -> Result { - let mut flow_info: mxl_sys::FlowInfo = unsafe { std::mem::zeroed() }; - unsafe { - Error::from_status( - self.instance - .api - .mxl_flow_reader_get_info(self.reader, &mut flow_info), - )?; - } - Ok(FlowInfo { value: flow_info }) - } - - pub fn get_complete_grain(&self, index: u64, timeout: Duration) -> Result { - let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; - let mut payload_ptr: *mut u8 = std::ptr::null_mut(); - let timeout_ns = timeout.as_nanos() as u64; - loop { - unsafe { - Error::from_status(self.instance.api.mxl_flow_reader_get_grain( - self.reader, - index, - timeout_ns, - &mut grain_info, - &mut payload_ptr, - ))?; - } - if grain_info.commitedSize != grain_info.grainSize { - // We don't need partial grains. Wait for the grain to be complete. - continue; - } - if payload_ptr.is_null() { - return Err(Error::Other(format!( - "Failed to get grain payload for index {}.", - index - ))); - } - break; - } - let payload = unsafe { - let slice = std::slice::from_raw_parts(payload_ptr, grain_info.grainSize as usize); - slice.to_vec() - }; - let user_data = grain_info.userData.to_vec(); - Ok(GrainData { user_data, payload }) - } -} - -impl Drop for MxlFlowReader<'_> { - fn drop(&mut self) { - if !self.reader.is_null() { - if let Err(error) = self.destroy() { - tracing::error!("Failed to release MXL flow reader: {:?}", error); - } - } - } -} From 9cb64fe75adbe654f214ed001d9f3e6ec2eb26ed Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 12:51:05 +0000 Subject: [PATCH 09/77] instance: don't use unwrap. Signed-off-by: Pedro Ferreira --- rust-bindings/README.md | 1 + rust-bindings/mxl/src/error.rs | 3 +++ rust-bindings/mxl/src/instance.rs | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rust-bindings/README.md b/rust-bindings/README.md index 24d9051bc..8dcb8a225 100644 --- a/rust-bindings/README.md +++ b/rust-bindings/README.md @@ -11,6 +11,7 @@ - Use `rustfmt` in it's default settings for code formatting. - The `cargo clippy` should be always clean. - Try to avoid adding more dependencies, unless really necessary. +- Never use `unwrap`, `expect`, or a similar construct that causes a panic. Always return errors. Tests are an exception. ## Building diff --git a/rust-bindings/mxl/src/error.rs b/rust-bindings/mxl/src/error.rs index 4c1327bbe..94ae3e3e0 100644 --- a/rust-bindings/mxl/src/error.rs +++ b/rust-bindings/mxl/src/error.rs @@ -26,6 +26,9 @@ pub enum Error { #[error("dlopen: {0}")] DlOpen(#[from] dlopen2::Error), + + #[error("Null string: {0}")] + NulString(#[from] std::ffi::NulError), } impl Error { diff --git a/rust-bindings/mxl/src/instance.rs b/rust-bindings/mxl/src/instance.rs index f44704ce2..f8321039f 100644 --- a/rust-bindings/mxl/src/instance.rs +++ b/rust-bindings/mxl/src/instance.rs @@ -13,8 +13,8 @@ impl MxlInstance { pub fn new(api: Container, domain: &str, options: &str) -> Result { let instance = unsafe { api.mxl_create_instance( - CString::new(domain).unwrap().as_ptr(), - CString::new(options).unwrap().as_ptr(), + CString::new(domain)?.as_ptr(), + CString::new(options)?.as_ptr(), ) }; if instance.is_null() { From 75fca90687128c867f4574349186081d1ac40256 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 12:53:39 +0000 Subject: [PATCH 10/77] instance: don't expose a destroy method as that breaks the invariants Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/src/instance.rs | 18 +----------------- rust-bindings/mxl/tests/basic_tests.rs | 1 - 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/rust-bindings/mxl/src/instance.rs b/rust-bindings/mxl/src/instance.rs index f8321039f..8a4660ddc 100644 --- a/rust-bindings/mxl/src/instance.rs +++ b/rust-bindings/mxl/src/instance.rs @@ -24,20 +24,6 @@ impl MxlInstance { } } - pub fn destroy(&mut self) -> Result<()> { - let result; - if self.instance.is_null() { - return Err(Error::Other( - "Internal instance not initialized.".to_string(), - )); - } - unsafe { - result = Error::from_status(self.api.mxl_destroy_instance(self.instance)); - } - self.instance = std::ptr::null_mut(); - result - } - pub fn create_flow_reader(&self, flow_id: &str) -> Result { let flow_id = CString::new(flow_id).map_err(|_| Error::InvalidArg)?; let options = CString::new("").map_err(|_| Error::InvalidArg)?; @@ -67,9 +53,7 @@ impl MxlInstance { impl Drop for MxlInstance { fn drop(&mut self) { if !self.instance.is_null() { - if let Err(error) = self.destroy() { - tracing::error!("Failed to destroy MXL instance: {:?}", error); - } + unsafe { self.api.mxl_destroy_instance(self.instance) }; } } } diff --git a/rust-bindings/mxl/tests/basic_tests.rs b/rust-bindings/mxl/tests/basic_tests.rs index bdd6f69cf..07ad503be 100644 --- a/rust-bindings/mxl/tests/basic_tests.rs +++ b/rust-bindings/mxl/tests/basic_tests.rs @@ -50,5 +50,4 @@ fn basic_mxl_writing_reading() { info!("Grain data len: {:?}", grain_data.payload.len()); flow_reader.destroy().unwrap(); drop(flow_reader); - mxl_instance.destroy().unwrap(); } From 68bef759e3095fa639533aeeaebaf94e6767fdd9 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 15:09:36 +0000 Subject: [PATCH 11/77] build: add a build.rs file to expose the mxl build path Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/build.rs | 23 +++++++++++++++++++++++ rust-bindings/mxl/src/config.rs | 10 ++++++++++ rust-bindings/mxl/src/lib.rs | 2 ++ rust-bindings/mxl/tests/basic_tests.rs | 11 +++-------- 4 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 rust-bindings/mxl/build.rs create mode 100644 rust-bindings/mxl/src/config.rs diff --git a/rust-bindings/mxl/build.rs b/rust-bindings/mxl/build.rs new file mode 100644 index 000000000..3ca88a2bd --- /dev/null +++ b/rust-bindings/mxl/build.rs @@ -0,0 +1,23 @@ +use std::env; +use std::path::PathBuf; + +#[cfg(debug_assertions)] +const BUILD_VARIANT: &str = "Linux-Clang-Debug"; +#[cfg(not(debug_assertions))] +const BUILD_VARIANT: &str = "Linux-Clang-Release"; + +fn main() { + let manifest_dir = + PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("failed to get current directory")); + let repo_root = manifest_dir.parent().unwrap().parent().unwrap(); + let build_dir = repo_root.join("build").join(BUILD_VARIANT); + + let out_path = PathBuf::from(env::var("OUT_DIR").expect("failed to get output directory")) + .join("constants.rs"); + + let data = format!( + "pub const MXL_BUILD_DIR: &str = \"{}\";\n", + build_dir.to_string_lossy() + ); + std::fs::write(out_path, data).expect("Unable to write file"); +} diff --git a/rust-bindings/mxl/src/config.rs b/rust-bindings/mxl/src/config.rs new file mode 100644 index 000000000..33e2d19b2 --- /dev/null +++ b/rust-bindings/mxl/src/config.rs @@ -0,0 +1,10 @@ +use std::str::FromStr; + +include!(concat!(env!("OUT_DIR"), "/constants.rs")); + +pub fn get_mxf_so_path() -> std::path::PathBuf { + std::path::PathBuf::from_str(MXL_BUILD_DIR) + .expect("build error: 'MXL_BUILD_DIR' is invalid") + .join("lib") + .join("libmxl.so") +} diff --git a/rust-bindings/mxl/src/lib.rs b/rust-bindings/mxl/src/lib.rs index d9e679ae8..40e32a908 100644 --- a/rust-bindings/mxl/src/lib.rs +++ b/rust-bindings/mxl/src/lib.rs @@ -4,6 +4,8 @@ mod flow; mod flow_reader; mod instance; +pub mod config; + pub use api::{MxlApi, load_api}; pub use error::{Error, Result}; pub use flow::*; diff --git a/rust-bindings/mxl/tests/basic_tests.rs b/rust-bindings/mxl/tests/basic_tests.rs index 07ad503be..0b983792f 100644 --- a/rust-bindings/mxl/tests/basic_tests.rs +++ b/rust-bindings/mxl/tests/basic_tests.rs @@ -2,9 +2,9 @@ /// /// The tests now require an MXL library of a specific name to be present in the system. This should /// change in the future. For now, feel free to just edit the path to your library. -use std::path::PathBuf; use std::time::Duration; +use mxl::config::get_mxf_so_path; use tracing::info; static LOG_ONCE: std::sync::Once = std::sync::Once::new(); @@ -22,12 +22,7 @@ fn setup_test() -> mxl::MxlInstance { .init(); }); - // To run the test, you need to alter the path below to point to a valid file with MXL - // implementation. It is possible to use just the file name, as long as the file is in the - // library search path. - let libpath = - PathBuf::from("/home/pac/Sources/MXL/mxl/install/Linux-Clang-Debug/lib/libmxl.so"); - let mxl_api = mxl::load_api(libpath).unwrap(); + let mxl_api = mxl::load_api(get_mxf_so_path()).unwrap(); // TODO: Randomize the domain name to allow parallel tests run. mxl::MxlInstance::new(mxl_api, "/dev/shm/mxl_domain", "").unwrap() } @@ -37,7 +32,7 @@ fn basic_mxl_writing_reading() { // TODO: Add the writing part of the test. Now the test requires the external MXL tool to write // data: `tools/mxl-gst/mxl-gst-videotestsrc -d /dev/shm/mxl_domain -f // ../../lib/tests/data/v210_flow.json` - let mut mxl_instance = setup_test(); + let mxl_instance = setup_test(); let mut flow_reader = mxl_instance .create_flow_reader("5fbec3b1-1b0f-417d-9059-8b94a47197ed") .unwrap(); From 7cb39174ee8672061d6ad610dc08fd688c6b52e7 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 15:41:04 +0000 Subject: [PATCH 12/77] build: use mxl headers directly Signed-off-by: Pedro Ferreira --- rust-bindings/mxl-sys/build.rs | 36 +- .../mxl-headers-2025-06-17/mxl/dataformat.h | 96 ------ .../mxl-sys/mxl-headers-2025-06-17/mxl/flow.h | 307 ------------------ .../mxl-headers-2025-06-17/mxl/flowinfo.h | 141 -------- .../mxl-sys/mxl-headers-2025-06-17/mxl/mxl.h | 84 ----- .../mxl-headers-2025-06-17/mxl/platform.h | 16 - .../mxl-headers-2025-06-17/mxl/rational.h | 23 -- .../mxl-sys/mxl-headers-2025-06-17/mxl/time.h | 77 ----- .../mxl-headers-2025-06-17/mxl/version.h | 7 - rust-bindings/mxl-sys/wrapper.h | 8 + 10 files changed, 25 insertions(+), 770 deletions(-) delete mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/dataformat.h delete mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flow.h delete mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flowinfo.h delete mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/mxl.h delete mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/platform.h delete mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/rational.h delete mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/time.h delete mode 100644 rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/version.h create mode 100644 rust-bindings/mxl-sys/wrapper.h diff --git a/rust-bindings/mxl-sys/build.rs b/rust-bindings/mxl-sys/build.rs index 7e6d25188..f9a7a6e3e 100644 --- a/rust-bindings/mxl-sys/build.rs +++ b/rust-bindings/mxl-sys/build.rs @@ -1,31 +1,29 @@ use std::env; use std::path::PathBuf; +#[cfg(debug_assertions)] +const BUILD_VARIANT: &str = "Linux-Clang-Debug"; +#[cfg(not(debug_assertions))] +const BUILD_VARIANT: &str = "Linux-Clang-Release"; + fn main() { - let headers_dir = "mxl-headers-2025-06-17"; - let headers = [ - "mxl/dataformat.h", - "mxl/flow.h", - "mxl/flowinfo.h", - "mxl/mxl.h", - "mxl/platform.h", - "mxl/rational.h", - "mxl/time.h", - "mxl/version.h", - ]; + let manifest_dir = + PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("failed to get current directory")); + let repo_root = manifest_dir.parent().unwrap().parent().unwrap(); + let build_dir = repo_root.join("build").join(BUILD_VARIANT); + let includes_dir = repo_root.join("lib").join("include"); + + let build_version_dir = build_dir.join("lib").join("include"); + let build_version_dir = build_version_dir.to_string_lossy(); + println!("cargo:include={build_version_dir}"); - let manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("failed to get current directory"); - let includes_dir = format!("{manifest_dir}/{headers_dir}"); + let includes_dir = includes_dir.to_string_lossy(); println!("cargo:include={includes_dir}"); let bindings = bindgen::builder() .clang_arg(format!("-I{includes_dir}")) - .headers( - headers - .iter() - .map(|val| format!("{includes_dir}/{val}")) - .collect::>(), - ) + .clang_arg(format!("-I{build_version_dir}")) + .header("wrapper.h") .derive_default(true) .derive_debug(true) .prepend_enum_name(false) diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/dataformat.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/dataformat.h deleted file mode 100644 index f30565689..000000000 --- a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/dataformat.h +++ /dev/null @@ -1,96 +0,0 @@ -#pragma once - -#ifdef __cplusplus -extern "C" -{ -#endif - /** - * Source and flow data formats as defined by AMWA NMOS IS-04, excluding `urn:x-nmos:format:data.event`. - */ - typedef enum mxlDataFormat - { - MXL_DATA_FORMAT_UNSPECIFIED, - MXL_DATA_FORMAT_VIDEO, - MXL_DATA_FORMAT_AUDIO, - MXL_DATA_FORMAT_DATA, - MXL_DATA_FORMAT_MUX, - } mxlDataFormat; - - /** - * Return whether the specified format is valid. - * \param[in] format the mxlDataFormat of interest. - * \return 1 if the format specified in \p format is valid, otherwise 0. - */ - inline int mxlIsValidDataFormat(int format) - { - switch (format) - { - case MXL_DATA_FORMAT_VIDEO: - case MXL_DATA_FORMAT_AUDIO: - case MXL_DATA_FORMAT_DATA: - case MXL_DATA_FORMAT_MUX: - return 1; - - default: - return 0; - } - } - - /** - * Return whether the specified format is supported by MXL. - * \param[in] format the mxlDataFormat of interest. - * \return 1 if the format specified in \p format is supported, otherwise 0. - */ - inline int mxlIsSupportedDataFormat(int format) - { - switch (format) - { - case MXL_DATA_FORMAT_VIDEO: - case MXL_DATA_FORMAT_AUDIO: - case MXL_DATA_FORMAT_DATA: - return 1; - - default: - return 0; - } - } - - /** - * Return whether the specified format is operating in discrete grains. - * \param[in] format the mxlDataFormat of interest. - * \return 1 if the format specified in \p format is operating with - * continuous samples, otherwise 0. - */ - inline int mxlIsDiscreteDataFormat(int format) - { - switch (format) - { - case MXL_DATA_FORMAT_VIDEO: - case MXL_DATA_FORMAT_DATA: - return 1; - - default: - return 0; - } - } - - /** - * Return whether the specified format is operating in continuous samples. - * \param[in] format the mxlDataFormat of interest. - * \return 1 if the format specified in \p format is operating with - * continuous samples, otherwise 0. - */ - inline int mxlIsContinuousDataFormat(int format) - { - switch (format) - { - case MXL_DATA_FORMAT_AUDIO: - return 1; - - default: - return 0; - } - } -#ifdef __cplusplus -} -#endif diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flow.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flow.h deleted file mode 100644 index 2d1ace473..000000000 --- a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flow.h +++ /dev/null @@ -1,307 +0,0 @@ -#pragma once - -#ifdef __cplusplus -# include -# include -#else -# include -# include -#endif - -#include -#include - -#ifdef __cplusplus -extern "C" -{ -#endif - -/* - * A grain can be marked as invalid for multiple reasons. for example, an input application may have - * timed out before receiving a grain in time, etc. Writing grain marked as invalid is the proper way - * to make the ringbuffer whilst letting consumers know that the grain is invalid. A consumer - * may choose to repeat the previous grain, insert silence, etc. - */ -#define GRAIN_FLAG_INVALID 0x00000001 // 1 << 0. - - /** - * The payload location of the grain - */ - typedef enum PayloadLocation - { - PAYLOAD_LOCATION_HOST_MEMORY = 0, - PAYLOAD_LOCATION_DEVICE_MEMORY = 1, - } PayloadLocation; - - /** - * A helper type used to describe consecutive sequences of bytes in memory. - */ - typedef struct BufferSlice - { - /** A pointer referring to the beginning of the slice. */ - void const* pointer; - /** The number of bytes that make up this slice. */ - size_t size; - } BufferSlice; - - /** - * A helper type used to describe consecutive sequences of mutable bytes in - * memory. - */ - typedef struct MutableBufferSlice - { - /** A pointer referring to the beginning of the slice. */ - void* pointer; - /** The number of bytes that make up this slice. */ - size_t size; - } MutableBufferSlice; - - /** - * A helper type used to describe consecutive sequences of bytes - * in a ring buffer that may potentially straddle the wrapraround - * point of the buffer. - */ - typedef struct WrappedBufferSlice - { - BufferSlice fragments[2]; - } WrappedBufferSlice; - - /** - * A helper type used to describe consecutive sequences of mutable bytes in - * a ring buffer that may potentially straddle the wrapraround point of the - * buffer. - */ - typedef struct MutableWrappedBufferSlice - { - MutableBufferSlice fragments[2]; - } MutableWrappedBufferSlice; - - /** - * A helper type used to describe consecutive sequences of bytes - * in memory in consecutive ring buffers separated by the specified - * stride of bytes. - */ - typedef struct WrappedMultiBufferSlice - { - WrappedBufferSlice base; - - /** - * The stride in bytes to get from a position in one buffer - * to the same position in the following buffer. - */ - size_t stride; - /** - * The number of buffers following the base buffer. - */ - size_t count; - } WrappedMultiBufferSlice; - - /** - * A helper type used to describe consecutive sequences of mutable bytes in - * memory in consecutive ring buffers separated by the specified stride of - * bytes. - */ - typedef struct MutableWrappedMultiBufferSlice - { - MutableWrappedBufferSlice base; - - /** - * The stride in bytes to get from a position in one buffer - * to the same position in the following buffer. - */ - size_t stride; - /** - * The number of buffers following the base buffer. - */ - size_t count; - } MutableWrappedMultiBufferSlice; - - - - typedef struct GrainInfo - { - /// Version of the structure. The only currently supported value is 1 - uint32_t version; - /// Size of the structure - uint32_t size; - /// Grain flags. - uint32_t flags; - /// Payload location - PayloadLocation payloadLocation; - /// Device index (if payload is in device memory). -1 if on host memory. - int32_t deviceIndex; - /// Size in bytes of the complete payload of a grain - uint32_t grainSize; - /// How many bytes in the grain are currently valid (commited). This is typically used when writing slices. - /// A grain is complete when commitedSize == grainSize - uint32_t commitedSize; - /// User data space - uint8_t userData[4068]; - } GrainInfo; - - typedef struct mxlFlowReader_t* mxlFlowReader; - typedef struct mxlFlowWriter_t* mxlFlowWriter; - - /// - /// Create a flow using a json flow definition - /// - /// \param[in] instance The mxl instance created using mxlCreateInstance - /// \param[in] flowDef A flow definition in the NMOS Flow json format. The flow ID is read from the field of this json object. - /// \param[in] options Additional options (undefined). \todo Specify and used the additional options. - /// \param[out] info A pointer to a FlowInfo structure. If not nullptr, this structure will be updated with the flow information after the flow is - /// created. -MXL_EXPORT - mxlStatus mxlCreateFlow(mxlInstance instance, char const* flowDef, char const* options, FlowInfo* info); - - MXL_EXPORT - mxlStatus mxlDestroyFlow(mxlInstance instance, char const* flowId); - - MXL_EXPORT - mxlStatus mxlCreateFlowReader(mxlInstance instance, char const* flowId, char const* options, mxlFlowReader* reader); - - MXL_EXPORT - mxlStatus mxlReleaseFlowReader(mxlInstance instance, mxlFlowReader reader); - - MXL_EXPORT - mxlStatus mxlCreateFlowWriter(mxlInstance instance, char const* flowId, char const* options, mxlFlowWriter* writer); - - MXL_EXPORT - mxlStatus mxlReleaseFlowWriter(mxlInstance instance, mxlFlowWriter writer); - - /** - * Get a copy of the current descriptive header of a Flow - * - * \param[in] reader A valid flow reader - * \param[out] info A valid pointer to a FlowInfo structure. on return, the structure will be updated with a copy of the current flow info value. - * \return The result code. \see mxlStatus - */ -MXL_EXPORT - mxlStatus mxlFlowReaderGetInfo(mxlFlowReader reader, FlowInfo* info); - - /** - * Accessors for a flow grain at a specific index - * - * \param[in] reader A valid discrete flow reader. - * \param[in] index The index of the grain to obtain - * \param[in] timeoutNs How long should we wait for the grain (in nanoseconds) - * \param[out] grain The requested GrainInfo structure. - * \param[out] payload The requested grain payload. - * \return The result code. \see mxlStatus - * \note Please note that this function can only be called on readers that - * operate on discrete flows. Any attempt to call this function on a - * reader that operates on another type of flow will result in an - * error. - */ -MXL_EXPORT - mxlStatus mxlFlowReaderGetGrain(mxlFlowReader reader, uint64_t index, uint64_t timeoutNs, GrainInfo* grain, - uint8_t** payload); - - /** - * Non-blocking accessors for a flow grain at a specific index - * - * \param[in] reader A valid flow reader - * \param[in] index The index of the grain to obtain - * \param[out] grain The requested GrainInfo structure. - * \param[out] payload The requested grain payload. - * \return The result code. \see mxlStatus - * \note Please note that this function can only be called on readers that - * operate on discrete flows. Any attempt to call this function on a - * reader that operates on another type of flow will result in an - * error. - */ -MXL_EXPORT - mxlStatus mxlFlowReaderGetGrainNonBlocking(mxlFlowReader reader, uint64_t index, GrainInfo* grain, - uint8_t** payload); - - /** - * Open a grain for mutation. The flow writer will remember which index is currently opened. Before opening a new grain - * for mutation, the user must either cancel the mutation using mxlFlowWriterCancelGrain or mxlFlowWriterCommitGrain. - * - * \todo Allow operating on multiple grains simultaneously, by making this function return a handle that has to be passed - * to mxlFlowWriterCommitGrain or mxlFlowWriterCancelGrain to identify the grain the call refers to. - * - * \param[in] writer A valid flow writer - * \param[in] index The index of the grain to obtain - * \param[out] grainInfo The requested GrainInfo structure. - * \param[out] payload The requested grain payload. - * \return The result code. \see mxlStatus - * \note Please note that this function can only be called on writers that - * operate on discrete flows. Any attempt to call this function on a - * writer that operates on another type of flow will result in an - * error. - */ -MXL_EXPORT - mxlStatus mxlFlowWriterOpenGrain(mxlFlowWriter writer, uint64_t index, GrainInfo* grainInfo, - uint8_t** payload); - - /** - * - * \param[in] writer A valid flow writer - */ -MXL_EXPORT - mxlStatus mxlFlowWriterCancelGrain(mxlFlowWriter writer); - - /** - * Inform mxl that a user is done writing the grain that was previously opened. This will in turn signal all readers waiting on the ringbuffer - * that a new grain is available. The graininfo flags field in shared memory will be updated based on grain->flags This will increase the head - * and potentially the tail IF this grain is the new head. - * - * \return The result code. \see mxlStatus - */ -MXL_EXPORT - mxlStatus mxlFlowWriterCommitGrain(mxlFlowWriter writer, GrainInfo const* grain); - - - /** - * Accessor for a specific set of samples across all channels starting at a - * specific index. - * - * \param[in] index The head index of the samples to obtain. - * \param[in] count The number of samples to obtain. - * \param[out] payloadBuffersSlices A pointer to a wrapped multi buffer - * slice that represents the requested range across all channel - * buffers. - * - * \return A status code describing the outcome of the call. - * \note No guarantees are made as to how long the caller may - * safely hang on to the returned range of samples without the - * risk of these samples being overwritten. - */ - MXL_EXPORT - mxlStatus mxlFlowReaderGetSamples(mxlFlowReader reader, uint64_t index, size_t count, WrappedMultiBufferSlice* payloadBuffersSlices); - - /** - * Open a specific set of mutable samples across all channels starting at a - * specific index for mutation. - * - * \param[in] index The head index of the samples that will be mutated. - * \param[in] count The number of samples in each channel that will be - * mutated. - * \param[out] payloadBuffersSlices A pointer to a mutable wrapped multi - * buffer slice that represents the requested range across all channel - * buffers. - * - * \return A status code describing the outcome of the call. - */ - MXL_EXPORT - mxlStatus mxlFlowWriterOpenSamples(mxlFlowWriter writer, uint64_t index, size_t count, MutableWrappedMultiBufferSlice* payloadBuffersSlices); - - /** - * Cancel the mutation of the previously opened range of samples. - * \param[in] writer A valid flow writer - * \return The result code. \see mxlStatus - */ - MXL_EXPORT - mxlStatus mxlFlowWriterCancelSamples(mxlFlowWriter writer); - - /** - * Inform mxl that a user is done writing the sample range that was previously opened. - * - * \param[in] writer A valid flow writer - * \return The result code. \see mxlStatus - */ - MXL_EXPORT - mxlStatus mxlFlowWriterCommitSamples(mxlFlowWriter writer); -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flowinfo.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flowinfo.h deleted file mode 100644 index 65d7fceb2..000000000 --- a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/flowinfo.h +++ /dev/null @@ -1,141 +0,0 @@ -#pragma once - -#ifdef __cplusplus -# include -#else -# include -#endif - -#include -#include - -#ifdef __cplusplus -extern "C" -{ -#endif - /** - * Metadata about media a flow that is independent of the data format of the - * flow and thus common to all flows handled by MXL. - */ - typedef struct CommonFlowInfo - { - /** The flow UUID. This should be identical to the {flowId} path component. */ - uint8_t id[16]; - - - /** The last time a producer wrote to the flow in nanoseconds since the epoch. */ - uint64_t lastWriteTime; - - /** The last time a consumer read from the flow in nanoseconds since the epoch. */ - uint64_t lastReadTime; - - - /** - * The data format of this flow. - * \see mxlDataFormat - */ - uint32_t format; - - /** No flags defined yet. */ - uint32_t flags; - - /** Reserved space for future extensions. */ - uint8_t reserved[80]; - } CommonFlowInfo; - - typedef struct DiscreteFlowInfo - { - /** - * The number of grains per second expressed as a rational. - * For VIDEO and DATA this value must match the 'grain_rate' found in the flow descriptor. - */ - Rational grainRate; - - /** - * How many grains in the ring buffer. This should be identical to the number of entries in the {mxlDomain}/{flowId}/grains/ folder. - * Accessing the shared memory section for that specific grain should be predictable. - */ - uint32_t grainCount; - - /** - * 32 bit word used syncronization between a writer and multiple readers. This value can be used by futexes. - * When a FlowWriter commits some data (a grain, a slice, etc) it will increment this value and then wake all FlowReaders waiting on this - * memory address. - */ - uint32_t syncCounter; - - /** The current head index of the ringbuffer. */ - uint64_t headIndex; - - /** Reserved space for future extensions. */ - uint8_t reserved[96]; - } DiscreteFlowInfo; - - typedef struct ContinuousFlowInfo - { - /** - * The number of samples per second in this continuous flow. - * For AUDIO flows this value must match the 'sample_rate' found in the flow descriptor. - */ - Rational sampleRate; - - /** - * The number of channels in this flow. - * A dedicated ring buffer is provided for each channel. - */ - uint32_t channelCount; - - /** - * The number of samples in each of the ring buffers. - */ - uint32_t bufferLength; - - /** - * The largest expected batch size in samples, in which new data is written to this this flow by its producer. - * This value must be less than half of the buffer length. - */ - uint32_t commitBatchSize; - - /** - * The largest expected batch size in samples, at which availability of new data is signaled to waiting consumers. - * This must be a multiple of the commit batch size greater or equal to 1. - * \todo Will quite probably be obsoleted by new timing model. - */ - uint32_t syncBatchSize; - - /** The current head index within the per channel ring buffers. */ - uint64_t headIndex; - - /** Reserved space for future extensions. */ - uint8_t reserved[88]; - } ContinuousFlowInfo; - - /** - * Binary structure stored in the Flow shared memory segment. - * The flow shared memory will be located in {mxlDomain}/{flowId} - * where {mxlDomain} is a filesystem location available to the application - */ - typedef struct FlowInfo - { - /** Version of this structure. The only currently supported value is 1 */ - uint32_t version; - - /** The total size of this structure */ - uint32_t size; - - CommonFlowInfo common; - - /** Format specific header data. */ - union - { - DiscreteFlowInfo discrete; - ContinuousFlowInfo continuous; - }; - - /** User data space. */ - uint8_t userData[3840]; - } FlowInfo; - -#ifdef __cplusplus -} -#endif diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/mxl.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/mxl.h deleted file mode 100644 index 9807bb7a7..000000000 --- a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/mxl.h +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once - -#ifdef __cplusplus -# include -#else -# include -#endif - -#include - -#ifdef __cplusplus -extern "C" -{ -#endif - - /// MXL SDK Status codes. - typedef enum mxlStatus - { - MXL_STATUS_OK, - MXL_ERR_UNKNOWN, - MXL_ERR_FLOW_NOT_FOUND, - MXL_ERR_OUT_OF_RANGE_TOO_LATE, - MXL_ERR_OUT_OF_RANGE_TOO_EARLY, - MXL_ERR_INVALID_FLOW_READER, - MXL_ERR_INVALID_FLOW_WRITER, - MXL_ERR_TIMEOUT, - MXL_ERR_INVALID_ARG, - MXL_ERR_CONFLICT, - } mxlStatus; - - /// MXL SDK Semantic versionning structure. - typedef struct mxlVersionType - { - uint16_t major; - uint16_t minor; - uint16_t bugfix; - uint16_t build; - } mxlVersionType; - - /// - /// Accessor for the version of the MXL SDK. - /// \param out_version Pointer to a mxlVersionType structure to be filled with the version information. - /// \return MXL_STATUS_OK if the version was successfully retrieved, MXL_ERR_INVALID_ARG if the pointer passed was NULL. - /// - MXL_EXPORT - mxlStatus mxlGetVersion(mxlVersionType* out_version); - - /** An opaque type representing an MXL instance. */ - typedef struct mxlInstance_t* mxlInstance; - - /// - /// Create a new MXL instance for a specific domain. - /// - /// \param in_mxlDomain The domain is the directory where the MXL ringbuffers files are stored. It should live on a tmpfs filesystem. - /// \param in_options Optional JSON string containing additional SDK options. Currently not used. - /// \return A pointer to the MXL instance or NULL if the instance could not be created. - /// - MXL_EXPORT - mxlInstance mxlCreateInstance(char const* in_mxlDomain, char const* in_options); - - /// - /// Iterates over all flows in the MXL domain and deletes any flows that are - /// no longer active (in other words, no readers or writers are using them. This typically - /// happens when an application creating flows writers crashes or exits without cleaning up. - /// A flow is considered active if a shared advisory lock is held on the data file of the flow. - /// This function is automatically called when the instance is created but should be called periodically - /// on a long running application to clean up any flows that are no longer active. - /// - MXL_EXPORT - mxlStatus mxlGarbageCollectFlows(mxlInstance in_instance); - - /// - /// Destroy the MXL instance. This will also release all flows readers/writers associated with the instance. - /// - /// \param in_instance The MXL instance to destroy. - /// \return MXL_STATUS_OK if the instance was successfully destroyed, MXL_ERR_INVALID_ARG if the pointer passed was NULL. MXL_ERR_UNKNOWN if an - /// error occurred during destruction. - /// - MXL_EXPORT - mxlStatus mxlDestroyInstance(mxlInstance in_instance); - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/platform.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/platform.h deleted file mode 100644 index 0d6f96ada..000000000 --- a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/platform.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#if defined(__GNUC__) || defined(__clang__) -# define MXL_EXPORT __attribute__((visibility("default"))) -#else -# define MXL_EXPORT -#endif - -// TODO: Tailor these more to specific language statdard levels -#ifdef __cplusplus -# define MXL_NODISCARD [[nodiscard]] -# define MXL_CONSTEXPR constexpr -#else -# define MXL_NODISCARD -# define MXL_CONSTEXPR inline -#endif diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/rational.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/rational.h deleted file mode 100644 index 9cb89bd09..000000000 --- a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/rational.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#ifdef __cplusplus -# include -#else -# include -#endif - -#ifdef __cplusplus -extern "C" -{ -#endif - - typedef struct Rational - { - int64_t numerator; - int64_t denominator; - } Rational; - -#ifdef __cplusplus -} -#endif - diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/time.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/time.h deleted file mode 100644 index 324467e52..000000000 --- a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/time.h +++ /dev/null @@ -1,77 +0,0 @@ -#pragma once - -#ifdef __cplusplus -# include -#else -# include -#endif - -#include -#include - -#ifdef __cplusplus -extern "C" -{ -#endif - -#define MXL_UNDEFINED_INDEX UINT64_MAX - - /** - * Get the current head index based on the current system TAI time - * Index 0 is defined to be index at the beginning of the epoch as defined by SMPTE ST 2059 - * - * \param[in] editRate The edit rate of the Flow - * \return The head index or MXL_UNDEFINED_INDEX if editRate is null or invalid - */ -MXL_EXPORT - uint64_t mxlGetCurrentHeadIndex(Rational const* editRate); - - /** - * Utility method to help compute how many nanoseconds we need to wait until the beginning of the specified index - * \param[in] index The head to wait for - * \param[in] editRate The edit rate of the Flow - * \return How many nanoseconds to wait or MXL_UNDEFINED_INDEX if editRate is null or invalid - */ -MXL_EXPORT - uint64_t mxlGetNsUntilHeadIndex(uint64_t index, Rational const* editRate); - - /** - * Get the current head index based on the user provided timespec - * Index 0 is defined to be index at the beginning of the epoch as defined by SMPTE ST 2059 - * - * \param[in] editRate The edit rate of the Flow - * \param[in] timestamp The time stamp in nanoseconds since the epoch. - * \return The head index or MXL_UNDEFINED_INDEX if editRate is null or invalid - */ -MXL_EXPORT - uint64_t mxlTimestampToHeadIndex(Rational const* editRate, uint64_t timestamp); - - /** - * Get the current timestamp based on the user provided head index. - * Index 0 is defined to be index at the beginning of the epoch as defined by SMPTE ST 2059 - * - * \param[in] editRate The edit rate of the Flow - * \param[in] index The head index of the flow. - * \return The time stamp in nanoseconds since the epoch or MXL_UNDEFINED_INDEX if editRate is null or invalid - */ -MXL_EXPORT - uint64_t mxlHeadIndexToTimestamp(Rational const* editRate, uint64_t index); - - /** - * Sleep for a specific amount of time. - * \param[in] ns How long to sleep for, in nanoseconds. - */ -MXL_EXPORT - void mxlSleepForNs(uint64_t ns); - - /** - * Get the current time using the most appropriate clock for the platform. - * - * \return The current time in nanoseconds since the epoch. - */ -MXL_EXPORT - uint64_t mxlGetTime(); - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/version.h b/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/version.h deleted file mode 100644 index ba2442a54..000000000 --- a/rust-bindings/mxl-sys/mxl-headers-2025-06-17/mxl/version.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -#define MXL_VERSION_MAJOR 0 -#define MXL_VERSION_MINOR 6 -#define MXL_VERSION_PATCH 0 -#define MXL_VERSION_BUILD 0 -#define MXL_VERSION "0.6.0.0" diff --git a/rust-bindings/mxl-sys/wrapper.h b/rust-bindings/mxl-sys/wrapper.h new file mode 100644 index 000000000..9969e9d1c --- /dev/null +++ b/rust-bindings/mxl-sys/wrapper.h @@ -0,0 +1,8 @@ +#include "mxl/dataformat.h" +#include "mxl/flow.h" +#include "mxl/flowinfo.h" +#include "mxl/mxl.h" +#include "mxl/platform.h" +#include "mxl/rational.h" +#include "mxl/time.h" +#include "mxl/version.h" From 4e33af49ac5e1c8913133e27d607e4d607cec88f Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 15:19:07 +0000 Subject: [PATCH 13/77] examples: add basic example Signed-off-by: Pedro Ferreira --- rust-bindings/Cargo.lock | 45 +++++++++++++++++++ rust-bindings/Cargo.toml | 10 +++-- rust-bindings/mxl/Cargo.toml | 1 + rust-bindings/mxl/examples/flow-reader.rs | 53 +++++++++++++++++++++++ 4 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 rust-bindings/mxl/examples/flow-reader.rs diff --git a/rust-bindings/Cargo.lock b/rust-bindings/Cargo.lock index 21f575c3b..1423a7750 100644 --- a/rust-bindings/Cargo.lock +++ b/rust-bindings/Cargo.lock @@ -80,6 +80,44 @@ dependencies = [ "libloading", ] +[[package]] +name = "clap" +version = "4.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_derive" +version = "4.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + [[package]] name = "dlopen2" version = "0.8.0" @@ -115,6 +153,12 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "itertools" version = "0.13.0" @@ -177,6 +221,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" name = "mxl" version = "0.1.0" dependencies = [ + "clap", "dlopen2", "mxl-sys", "thiserror", diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml index 77df3d4a3..275c009ba 100644 --- a/rust-bindings/Cargo.toml +++ b/rust-bindings/Cargo.toml @@ -1,8 +1,5 @@ [workspace] -members = [ - "mxl", - "mxl-sys", -] +members = ["mxl", "mxl-sys"] resolver = "2" @@ -22,6 +19,11 @@ thiserror = "2.0.12" tracing = { version = "0.1", features = ["log"] } tracing-subscriber = { version = "0.3", features = ["env-filter", "std"] } +[workspace.dependencies.clap] +version = "4.1.4" +default-features = false +features = ["std", "derive", "cargo", "env", "help", "usage", "error-context"] + [profile.release-with-debug] debug = true inherits = "release" diff --git a/rust-bindings/mxl/Cargo.toml b/rust-bindings/mxl/Cargo.toml index fef324986..06f9c3528 100644 --- a/rust-bindings/mxl/Cargo.toml +++ b/rust-bindings/mxl/Cargo.toml @@ -12,4 +12,5 @@ thiserror.workspace = true tracing.workspace = true [dev-dependencies] +clap.workspace = true tracing-subscriber.workspace = true diff --git a/rust-bindings/mxl/examples/flow-reader.rs b/rust-bindings/mxl/examples/flow-reader.rs new file mode 100644 index 000000000..3f7ce597a --- /dev/null +++ b/rust-bindings/mxl/examples/flow-reader.rs @@ -0,0 +1,53 @@ +use std::time::Duration; + +use clap::Parser; +use mxl::config::get_mxf_so_path; +use tracing::info; + +const READ_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Debug, Parser)] +#[command(version = clap::crate_version!(), author = clap::crate_authors!())] +pub struct Opts { + /// The path to the shmem directory where the mxl domain is mapped. + #[arg(long)] + pub mxl_domain: String, + + /// The id of the flow. + #[arg(long)] + pub flow_id: String, +} + +fn main() -> Result<(), mxl::Error> { + setup_logging(); + let opts: Opts = Opts::parse(); + + let mxl_api = mxl::load_api(get_mxf_so_path())?; + let mxl_instance = mxl::MxlInstance::new(mxl_api, &opts.mxl_domain, "")?; + let reader = mxl_instance.create_flow_reader(&opts.flow_id)?; + let flow_info = reader.get_info()?; + let rate = flow_info.discrete_flow_info()?.grainRate; + let current_head_index = mxl_instance.get_current_head_index(&rate); + + info!("Grain rate: {}/{}", rate.numerator, rate.denominator); + + for index in current_head_index.. { + let grain_data = reader.get_complete_grain(index, READ_TIMEOUT)?; + info!( + "Index: {index} Grain data len: {:?}", + grain_data.payload.len() + ); + } + + Ok(()) +} + +fn setup_logging() { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::builder() + .with_default_directive(tracing::level_filters::LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .init(); +} From 65d1822bdd022674e7753d1b5a5d64ed2bd9e647 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 24 Jun 2025 07:54:54 +0000 Subject: [PATCH 14/77] reader: decouple lifetime from instance Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/src/flow.rs | 2 +- rust-bindings/mxl/src/flow_reader.rs | 27 ++++++++------ rust-bindings/mxl/src/instance.rs | 50 +++++++++++++++----------- rust-bindings/mxl/tests/basic_tests.rs | 1 - 4 files changed, 47 insertions(+), 33 deletions(-) diff --git a/rust-bindings/mxl/src/flow.rs b/rust-bindings/mxl/src/flow.rs index 6172a764a..bb6b838fb 100644 --- a/rust-bindings/mxl/src/flow.rs +++ b/rust-bindings/mxl/src/flow.rs @@ -34,7 +34,7 @@ impl FlowInfo { && self.value.common.format != mxl_sys::MXL_DATA_FORMAT_DATA { return Err(Error::Other(format!( - "Flow format is {}, video or data require.", + "Flow format is {}, video or data required.", self.value.common.format ))); } diff --git a/rust-bindings/mxl/src/flow_reader.rs b/rust-bindings/mxl/src/flow_reader.rs index 69b9a7805..e1a299ba2 100644 --- a/rust-bindings/mxl/src/flow_reader.rs +++ b/rust-bindings/mxl/src/flow_reader.rs @@ -1,16 +1,21 @@ -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use crate::{ - Error, MxlInstance, Result, + Error, Result, flow::{FlowInfo, GrainData}, + instance::InstanceContext, }; -pub struct MxlFlowReader<'a> { - pub(crate) instance: &'a MxlInstance, - pub(crate) reader: mxl_sys::mxlFlowReader, +pub struct MxlFlowReader { + context: Arc, + reader: mxl_sys::mxlFlowReader, } -impl<'a> MxlFlowReader<'a> { +impl MxlFlowReader { + pub(crate) fn new(context: Arc, reader: mxl_sys::mxlFlowReader) -> Self { + Self { context, reader } + } + pub fn destroy(&mut self) -> Result<()> { let result; if self.reader.is_null() { @@ -18,9 +23,9 @@ impl<'a> MxlFlowReader<'a> { } unsafe { result = Error::from_status( - self.instance + self.context .api - .mxl_release_flow_reader(self.instance.instance, self.reader), + .mxl_release_flow_reader(self.context.instance, self.reader), ); } self.reader = std::ptr::null_mut(); @@ -31,7 +36,7 @@ impl<'a> MxlFlowReader<'a> { let mut flow_info: mxl_sys::FlowInfo = unsafe { std::mem::zeroed() }; unsafe { Error::from_status( - self.instance + self.context .api .mxl_flow_reader_get_info(self.reader, &mut flow_info), )?; @@ -45,7 +50,7 @@ impl<'a> MxlFlowReader<'a> { let timeout_ns = timeout.as_nanos() as u64; loop { unsafe { - Error::from_status(self.instance.api.mxl_flow_reader_get_grain( + Error::from_status(self.context.api.mxl_flow_reader_get_grain( self.reader, index, timeout_ns, @@ -74,7 +79,7 @@ impl<'a> MxlFlowReader<'a> { } } -impl Drop for MxlFlowReader<'_> { +impl Drop for MxlFlowReader { fn drop(&mut self) { if !self.reader.is_null() { if let Err(error) = self.destroy() { diff --git a/rust-bindings/mxl/src/instance.rs b/rust-bindings/mxl/src/instance.rs index 8a4660ddc..3ceefea61 100644 --- a/rust-bindings/mxl/src/instance.rs +++ b/rust-bindings/mxl/src/instance.rs @@ -1,14 +1,34 @@ -use std::ffi::CString; +use std::{ffi::CString, sync::Arc}; use dlopen2::wrapper::Container; use crate::{Error, MxlApi, MxlFlowReader, Result}; -pub struct MxlInstance { +/// This struct stores the context that is shared by all objects. +/// It is separated out from `MxlInstance` so that it can be cloned +/// and other objects' lifetimes be decoupled from the MxlInstance +/// itself. +pub(crate) struct InstanceContext { pub(crate) api: Container, pub(crate) instance: mxl_sys::mxlInstance, } +// Allow sharing the context across threads and tasks freely. +unsafe impl Send for InstanceContext {} +unsafe impl Sync for InstanceContext {} + +impl Drop for InstanceContext { + fn drop(&mut self) { + if !self.instance.is_null() { + unsafe { self.api.mxl_destroy_instance(self.instance) }; + } + } +} + +pub struct MxlInstance { + context: Arc, +} + impl MxlInstance { pub fn new(api: Container, domain: &str, options: &str) -> Result { let instance = unsafe { @@ -20,17 +40,18 @@ impl MxlInstance { if instance.is_null() { Err(Error::Other("Failed to create MXL instance.".to_string())) } else { - Ok(Self { api, instance }) + let context = Arc::new(InstanceContext { api, instance }); + Ok(Self { context }) } } pub fn create_flow_reader(&self, flow_id: &str) -> Result { - let flow_id = CString::new(flow_id).map_err(|_| Error::InvalidArg)?; - let options = CString::new("").map_err(|_| Error::InvalidArg)?; + let flow_id = CString::new(flow_id)?; + let options = CString::new("")?; let mut reader: mxl_sys::mxlFlowReader = std::ptr::null_mut(); unsafe { - Error::from_status(self.api.mxl_create_flow_reader( - self.instance, + Error::from_status(self.context.api.mxl_create_flow_reader( + self.context.instance, flow_id.as_ptr(), options.as_ptr(), &mut reader, @@ -39,21 +60,10 @@ impl MxlInstance { if reader.is_null() { return Err(Error::Other("Failed to create flow reader.".to_string())); } - Ok(MxlFlowReader { - instance: self, - reader, - }) + Ok(MxlFlowReader::new(self.context.clone(), reader)) } pub fn get_current_head_index(&self, rational: &mxl_sys::Rational) -> u64 { - unsafe { self.api.mxl_get_current_head_index(rational) } - } -} - -impl Drop for MxlInstance { - fn drop(&mut self) { - if !self.instance.is_null() { - unsafe { self.api.mxl_destroy_instance(self.instance) }; - } + unsafe { self.context.api.mxl_get_current_head_index(rational) } } } diff --git a/rust-bindings/mxl/tests/basic_tests.rs b/rust-bindings/mxl/tests/basic_tests.rs index 0b983792f..55306d679 100644 --- a/rust-bindings/mxl/tests/basic_tests.rs +++ b/rust-bindings/mxl/tests/basic_tests.rs @@ -44,5 +44,4 @@ fn basic_mxl_writing_reading() { .unwrap(); info!("Grain data len: {:?}", grain_data.payload.len()); flow_reader.destroy().unwrap(); - drop(flow_reader); } From f5a47438939591faeeeddd67936a1f13cee7461a Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 24 Jun 2025 07:57:21 +0000 Subject: [PATCH 15/77] instance: add a destroy function for testing Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/src/instance.rs | 27 ++++++++++++++++++++++++++ rust-bindings/mxl/tests/basic_tests.rs | 1 + 2 files changed, 28 insertions(+) diff --git a/rust-bindings/mxl/src/instance.rs b/rust-bindings/mxl/src/instance.rs index 3ceefea61..07a9e1091 100644 --- a/rust-bindings/mxl/src/instance.rs +++ b/rust-bindings/mxl/src/instance.rs @@ -17,6 +17,21 @@ pub(crate) struct InstanceContext { unsafe impl Send for InstanceContext {} unsafe impl Sync for InstanceContext {} +impl InstanceContext { + /// This function forces the destruction of the MXL instance. + /// It is not safe as other objects may still be using it. + /// It is meant for testing purposes only. + /// + /// # Safety + /// + /// The caller must ensure that no other objects are using the MXL instance when this function is called. + /// Calling this function while other references exist may lead to undefined behavior. + pub unsafe fn destroy(&self) -> Result<()> { + unsafe { self.api.mxl_destroy_instance(self.instance) }; + Ok(()) + } +} + impl Drop for InstanceContext { fn drop(&mut self) { if !self.instance.is_null() { @@ -66,4 +81,16 @@ impl MxlInstance { pub fn get_current_head_index(&self, rational: &mxl_sys::Rational) -> u64 { unsafe { self.context.api.mxl_get_current_head_index(rational) } } + + /// This function forces the destruction of the MXL instance. + /// It is not safe as other objects may still be using it. + /// It is meant for testing purposes only. + /// + /// # Safety + /// + /// The caller must ensure that no other objects are using the MXL instance when this function is called. + /// Calling this function while other references exist may lead to undefined behavior. + pub unsafe fn destroy(self) -> Result<()> { + unsafe { self.context.destroy() } + } } diff --git a/rust-bindings/mxl/tests/basic_tests.rs b/rust-bindings/mxl/tests/basic_tests.rs index 55306d679..717af6e67 100644 --- a/rust-bindings/mxl/tests/basic_tests.rs +++ b/rust-bindings/mxl/tests/basic_tests.rs @@ -44,4 +44,5 @@ fn basic_mxl_writing_reading() { .unwrap(); info!("Grain data len: {:?}", grain_data.payload.len()); flow_reader.destroy().unwrap(); + unsafe { mxl_instance.destroy() }.unwrap(); } From c1d5550a4e39aeb599fa509cc5a3be76370aba1c Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 24 Jun 2025 07:32:18 +0000 Subject: [PATCH 16/77] reader: consume self on destroy so that the invariants are not broken Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/src/flow_reader.rs | 41 +++++++++++++++----------- rust-bindings/mxl/tests/basic_tests.rs | 2 +- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/rust-bindings/mxl/src/flow_reader.rs b/rust-bindings/mxl/src/flow_reader.rs index e1a299ba2..f39ff61f5 100644 --- a/rust-bindings/mxl/src/flow_reader.rs +++ b/rust-bindings/mxl/src/flow_reader.rs @@ -16,20 +16,8 @@ impl MxlFlowReader { Self { context, reader } } - pub fn destroy(&mut self) -> Result<()> { - let result; - if self.reader.is_null() { - return Err(Error::InvalidArg); - } - unsafe { - result = Error::from_status( - self.context - .api - .mxl_release_flow_reader(self.context.instance, self.reader), - ); - } - self.reader = std::ptr::null_mut(); - result + pub fn destroy(mut self) -> Result<()> { + self.destroy_inner() } pub fn get_info(&self) -> Result { @@ -77,14 +65,33 @@ impl MxlFlowReader { let user_data = grain_info.userData.to_vec(); Ok(GrainData { user_data, payload }) } + + fn destroy_inner(&mut self) -> Result<()> { + if self.reader.is_null() { + return Err(Error::InvalidArg); + } + + let mut reader = std::ptr::null_mut(); + std::mem::swap(&mut self.reader, &mut reader); + + let result = Error::from_status(unsafe { + self.context + .api + .mxl_release_flow_reader(self.context.instance, reader) + }); + + if let Err(err) = &result { + tracing::error!("Failed to release MXL flow reader: {:?}", err); + } + + result + } } impl Drop for MxlFlowReader { fn drop(&mut self) { if !self.reader.is_null() { - if let Err(error) = self.destroy() { - tracing::error!("Failed to release MXL flow reader: {:?}", error); - } + let _ = self.destroy_inner(); } } } diff --git a/rust-bindings/mxl/tests/basic_tests.rs b/rust-bindings/mxl/tests/basic_tests.rs index 717af6e67..34707220d 100644 --- a/rust-bindings/mxl/tests/basic_tests.rs +++ b/rust-bindings/mxl/tests/basic_tests.rs @@ -33,7 +33,7 @@ fn basic_mxl_writing_reading() { // data: `tools/mxl-gst/mxl-gst-videotestsrc -d /dev/shm/mxl_domain -f // ../../lib/tests/data/v210_flow.json` let mxl_instance = setup_test(); - let mut flow_reader = mxl_instance + let flow_reader = mxl_instance .create_flow_reader("5fbec3b1-1b0f-417d-9059-8b94a47197ed") .unwrap(); let flow_info = flow_reader.get_info().unwrap(); From 88fb975576d04c8dc2f38b9a6a1ba5a796bd6684 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Sun, 22 Jun 2025 19:18:05 +0000 Subject: [PATCH 17/77] grain data: don't allocate and copy payload & user data Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/src/flow.rs | 5 ----- rust-bindings/mxl/src/flow_reader.rs | 32 ++++++++++++++++++---------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/rust-bindings/mxl/src/flow.rs b/rust-bindings/mxl/src/flow.rs index bb6b838fb..7b293f629 100644 --- a/rust-bindings/mxl/src/flow.rs +++ b/rust-bindings/mxl/src/flow.rs @@ -41,8 +41,3 @@ impl FlowInfo { Ok(unsafe { &self.value.__bindgen_anon_1.discrete }) } } - -pub struct GrainData { - pub user_data: Vec, - pub payload: Vec, -} diff --git a/rust-bindings/mxl/src/flow_reader.rs b/rust-bindings/mxl/src/flow_reader.rs index f39ff61f5..13e9fc6a2 100644 --- a/rust-bindings/mxl/src/flow_reader.rs +++ b/rust-bindings/mxl/src/flow_reader.rs @@ -1,10 +1,11 @@ use std::{sync::Arc, time::Duration}; -use crate::{ - Error, Result, - flow::{FlowInfo, GrainData}, - instance::InstanceContext, -}; +use crate::{Error, Result, flow::FlowInfo, instance::InstanceContext}; + +pub struct GrainData<'a> { + pub user_data: &'a [u8], + pub payload: &'a [u8], +} pub struct MxlFlowReader { context: Arc, @@ -32,7 +33,11 @@ impl MxlFlowReader { Ok(FlowInfo { value: flow_info }) } - pub fn get_complete_grain(&self, index: u64, timeout: Duration) -> Result { + pub fn get_complete_grain<'a>( + &'a self, + index: u64, + timeout: Duration, + ) -> Result> { let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; let mut payload_ptr: *mut u8 = std::ptr::null_mut(); let timeout_ns = timeout.as_nanos() as u64; @@ -58,11 +63,16 @@ impl MxlFlowReader { } break; } - let payload = unsafe { - let slice = std::slice::from_raw_parts(payload_ptr, grain_info.grainSize as usize); - slice.to_vec() - }; - let user_data = grain_info.userData.to_vec(); + + // SAFETY + // We know that the lifetime is as long as the flow, so it is at least self's lifetime. + // It may happen that the buffer is overwritten by a subsequent write, but it is safe. + let user_data: &'a [u8] = + unsafe { std::mem::transmute::<&[u8], &'a [u8]>(&grain_info.userData) }; + + let payload = + unsafe { std::slice::from_raw_parts(payload_ptr, grain_info.grainSize as usize) }; + Ok(GrainData { user_data, payload }) } From cd07baee9d9ce4202a8c1c1851713868095e323f Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Mon, 23 Jun 2025 18:48:26 +0000 Subject: [PATCH 18/77] error: preserve the error code for unknown errors Signed-off-by: Pedro Ferreira --- rust-bindings/mxl/src/error.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rust-bindings/mxl/src/error.rs b/rust-bindings/mxl/src/error.rs index 94ae3e3e0..a56a4aad6 100644 --- a/rust-bindings/mxl/src/error.rs +++ b/rust-bindings/mxl/src/error.rs @@ -2,8 +2,8 @@ pub type Result = core::result::Result; #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("Unknown error")] - Unknown, + #[error("Unknown error: {0}")] + Unknown(mxl_sys::mxlStatus), #[error("Flow not found")] FlowNotFound, #[error("Out of range - too late")] @@ -35,7 +35,7 @@ impl Error { pub fn from_status(status: mxl_sys::mxlStatus) -> Result<()> { match status { mxl_sys::MXL_STATUS_OK => Ok(()), - mxl_sys::MXL_ERR_UNKNOWN => Err(Error::Unknown), + mxl_sys::MXL_ERR_UNKNOWN => Err(Error::Unknown(mxl_sys::MXL_ERR_UNKNOWN)), mxl_sys::MXL_ERR_FLOW_NOT_FOUND => Err(Error::FlowNotFound), mxl_sys::MXL_ERR_OUT_OF_RANGE_TOO_LATE => Err(Error::OutOfRangeTooLate), mxl_sys::MXL_ERR_OUT_OF_RANGE_TOO_EARLY => Err(Error::OutOfRangeTooEarly), @@ -44,7 +44,7 @@ impl Error { mxl_sys::MXL_ERR_TIMEOUT => Err(Error::Timeout), mxl_sys::MXL_ERR_INVALID_ARG => Err(Error::InvalidArg), mxl_sys::MXL_ERR_CONFLICT => Err(Error::Conflict), - _ => Err(Error::Unknown), + other => Err(Error::Unknown(other)), } } } From 03a750595d5ed4aa5df0664b60a88683dc40e37c Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 24 Jun 2025 09:07:36 +0000 Subject: [PATCH 19/77] layout: rename rust-bindings to rust Signed-off-by: Pedro Ferreira --- .vscode/settings.json | 2 +- {rust-bindings => rust}/.gitattributes | 0 {rust-bindings => rust}/.gitignore | 0 {rust-bindings => rust}/Cargo.lock | 0 {rust-bindings => rust}/Cargo.toml | 0 {rust-bindings => rust}/README.md | 0 {rust-bindings => rust}/mxl-sys/Cargo.toml | 0 {rust-bindings => rust}/mxl-sys/build.rs | 0 {rust-bindings => rust}/mxl-sys/src/lib.rs | 0 {rust-bindings => rust}/mxl-sys/tests/simple_test.rs | 0 {rust-bindings => rust}/mxl-sys/wrapper.h | 0 {rust-bindings => rust}/mxl/Cargo.toml | 0 {rust-bindings => rust}/mxl/build.rs | 0 {rust-bindings => rust}/mxl/examples/flow-reader.rs | 0 {rust-bindings => rust}/mxl/src/api.rs | 0 {rust-bindings => rust}/mxl/src/config.rs | 0 {rust-bindings => rust}/mxl/src/error.rs | 0 {rust-bindings => rust}/mxl/src/flow.rs | 0 {rust-bindings => rust}/mxl/src/flow_reader.rs | 0 {rust-bindings => rust}/mxl/src/instance.rs | 0 {rust-bindings => rust}/mxl/src/lib.rs | 0 {rust-bindings => rust}/mxl/tests/basic_tests.rs | 0 22 files changed, 1 insertion(+), 1 deletion(-) rename {rust-bindings => rust}/.gitattributes (100%) rename {rust-bindings => rust}/.gitignore (100%) rename {rust-bindings => rust}/Cargo.lock (100%) rename {rust-bindings => rust}/Cargo.toml (100%) rename {rust-bindings => rust}/README.md (100%) rename {rust-bindings => rust}/mxl-sys/Cargo.toml (100%) rename {rust-bindings => rust}/mxl-sys/build.rs (100%) rename {rust-bindings => rust}/mxl-sys/src/lib.rs (100%) rename {rust-bindings => rust}/mxl-sys/tests/simple_test.rs (100%) rename {rust-bindings => rust}/mxl-sys/wrapper.h (100%) rename {rust-bindings => rust}/mxl/Cargo.toml (100%) rename {rust-bindings => rust}/mxl/build.rs (100%) rename {rust-bindings => rust}/mxl/examples/flow-reader.rs (100%) rename {rust-bindings => rust}/mxl/src/api.rs (100%) rename {rust-bindings => rust}/mxl/src/config.rs (100%) rename {rust-bindings => rust}/mxl/src/error.rs (100%) rename {rust-bindings => rust}/mxl/src/flow.rs (100%) rename {rust-bindings => rust}/mxl/src/flow_reader.rs (100%) rename {rust-bindings => rust}/mxl/src/instance.rs (100%) rename {rust-bindings => rust}/mxl/src/lib.rs (100%) rename {rust-bindings => rust}/mxl/tests/basic_tests.rs (100%) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9ab20c015..93f997e37 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 { "rust-analyzer.linkedProjects": [ - "rust-bindings/Cargo.toml" + "rust/Cargo.toml" ], "files.associations": { "array": "cpp", diff --git a/rust-bindings/.gitattributes b/rust/.gitattributes similarity index 100% rename from rust-bindings/.gitattributes rename to rust/.gitattributes diff --git a/rust-bindings/.gitignore b/rust/.gitignore similarity index 100% rename from rust-bindings/.gitignore rename to rust/.gitignore diff --git a/rust-bindings/Cargo.lock b/rust/Cargo.lock similarity index 100% rename from rust-bindings/Cargo.lock rename to rust/Cargo.lock diff --git a/rust-bindings/Cargo.toml b/rust/Cargo.toml similarity index 100% rename from rust-bindings/Cargo.toml rename to rust/Cargo.toml diff --git a/rust-bindings/README.md b/rust/README.md similarity index 100% rename from rust-bindings/README.md rename to rust/README.md diff --git a/rust-bindings/mxl-sys/Cargo.toml b/rust/mxl-sys/Cargo.toml similarity index 100% rename from rust-bindings/mxl-sys/Cargo.toml rename to rust/mxl-sys/Cargo.toml diff --git a/rust-bindings/mxl-sys/build.rs b/rust/mxl-sys/build.rs similarity index 100% rename from rust-bindings/mxl-sys/build.rs rename to rust/mxl-sys/build.rs diff --git a/rust-bindings/mxl-sys/src/lib.rs b/rust/mxl-sys/src/lib.rs similarity index 100% rename from rust-bindings/mxl-sys/src/lib.rs rename to rust/mxl-sys/src/lib.rs diff --git a/rust-bindings/mxl-sys/tests/simple_test.rs b/rust/mxl-sys/tests/simple_test.rs similarity index 100% rename from rust-bindings/mxl-sys/tests/simple_test.rs rename to rust/mxl-sys/tests/simple_test.rs diff --git a/rust-bindings/mxl-sys/wrapper.h b/rust/mxl-sys/wrapper.h similarity index 100% rename from rust-bindings/mxl-sys/wrapper.h rename to rust/mxl-sys/wrapper.h diff --git a/rust-bindings/mxl/Cargo.toml b/rust/mxl/Cargo.toml similarity index 100% rename from rust-bindings/mxl/Cargo.toml rename to rust/mxl/Cargo.toml diff --git a/rust-bindings/mxl/build.rs b/rust/mxl/build.rs similarity index 100% rename from rust-bindings/mxl/build.rs rename to rust/mxl/build.rs diff --git a/rust-bindings/mxl/examples/flow-reader.rs b/rust/mxl/examples/flow-reader.rs similarity index 100% rename from rust-bindings/mxl/examples/flow-reader.rs rename to rust/mxl/examples/flow-reader.rs diff --git a/rust-bindings/mxl/src/api.rs b/rust/mxl/src/api.rs similarity index 100% rename from rust-bindings/mxl/src/api.rs rename to rust/mxl/src/api.rs diff --git a/rust-bindings/mxl/src/config.rs b/rust/mxl/src/config.rs similarity index 100% rename from rust-bindings/mxl/src/config.rs rename to rust/mxl/src/config.rs diff --git a/rust-bindings/mxl/src/error.rs b/rust/mxl/src/error.rs similarity index 100% rename from rust-bindings/mxl/src/error.rs rename to rust/mxl/src/error.rs diff --git a/rust-bindings/mxl/src/flow.rs b/rust/mxl/src/flow.rs similarity index 100% rename from rust-bindings/mxl/src/flow.rs rename to rust/mxl/src/flow.rs diff --git a/rust-bindings/mxl/src/flow_reader.rs b/rust/mxl/src/flow_reader.rs similarity index 100% rename from rust-bindings/mxl/src/flow_reader.rs rename to rust/mxl/src/flow_reader.rs diff --git a/rust-bindings/mxl/src/instance.rs b/rust/mxl/src/instance.rs similarity index 100% rename from rust-bindings/mxl/src/instance.rs rename to rust/mxl/src/instance.rs diff --git a/rust-bindings/mxl/src/lib.rs b/rust/mxl/src/lib.rs similarity index 100% rename from rust-bindings/mxl/src/lib.rs rename to rust/mxl/src/lib.rs diff --git a/rust-bindings/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs similarity index 100% rename from rust-bindings/mxl/tests/basic_tests.rs rename to rust/mxl/tests/basic_tests.rs From 4a604cc73a20a832047a966def1d3f701d367454 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 24 Jun 2025 10:15:28 +0000 Subject: [PATCH 20/77] grain data: add OwnedGrainData and conversion from GrainData Signed-off-by: Pedro Ferreira --- rust/mxl/src/flow_reader.rs | 7 +------ rust/mxl/src/grain_data.rs | 36 +++++++++++++++++++++++++++++++++++ rust/mxl/src/lib.rs | 2 ++ rust/mxl/tests/basic_tests.rs | 3 ++- 4 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 rust/mxl/src/grain_data.rs diff --git a/rust/mxl/src/flow_reader.rs b/rust/mxl/src/flow_reader.rs index 13e9fc6a2..51d1d2d52 100644 --- a/rust/mxl/src/flow_reader.rs +++ b/rust/mxl/src/flow_reader.rs @@ -1,11 +1,6 @@ use std::{sync::Arc, time::Duration}; -use crate::{Error, Result, flow::FlowInfo, instance::InstanceContext}; - -pub struct GrainData<'a> { - pub user_data: &'a [u8], - pub payload: &'a [u8], -} +use crate::{Error, GrainData, Result, flow::FlowInfo, instance::InstanceContext}; pub struct MxlFlowReader { context: Arc, diff --git a/rust/mxl/src/grain_data.rs b/rust/mxl/src/grain_data.rs new file mode 100644 index 000000000..fabdd1354 --- /dev/null +++ b/rust/mxl/src/grain_data.rs @@ -0,0 +1,36 @@ +pub struct GrainData<'a> { + pub user_data: &'a [u8], + pub payload: &'a [u8], +} + +impl<'a> GrainData<'a> { + pub fn to_owned(&self) -> OwnedGrainData { + self.into() + } +} + +impl<'a> AsRef> for GrainData<'a> { + fn as_ref(&self) -> &GrainData<'a> { + self + } +} + +pub struct OwnedGrainData { + pub user_data: Vec, + pub payload: Vec, +} + +impl<'a> From<&GrainData<'a>> for OwnedGrainData { + fn from(value: &GrainData<'a>) -> Self { + Self { + user_data: value.user_data.to_vec(), + payload: value.payload.to_vec(), + } + } +} + +impl<'a> From> for OwnedGrainData { + fn from(value: GrainData<'a>) -> Self { + value.as_ref().into() + } +} diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index 40e32a908..3c388c6dd 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -2,6 +2,7 @@ mod api; mod error; mod flow; mod flow_reader; +mod grain_data; mod instance; pub mod config; @@ -10,4 +11,5 @@ pub use api::{MxlApi, load_api}; pub use error::{Error, Result}; pub use flow::*; pub use flow_reader::MxlFlowReader; +pub use grain_data::*; pub use instance::MxlInstance; diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index 34707220d..7a1ba8560 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -4,7 +4,7 @@ /// change in the future. For now, feel free to just edit the path to your library. use std::time::Duration; -use mxl::config::get_mxf_so_path; +use mxl::{OwnedGrainData, config::get_mxf_so_path}; use tracing::info; static LOG_ONCE: std::sync::Once = std::sync::Once::new(); @@ -42,6 +42,7 @@ fn basic_mxl_writing_reading() { let grain_data = flow_reader .get_complete_grain(current_head_index, Duration::from_secs(5)) .unwrap(); + let grain_data: OwnedGrainData = grain_data.into(); info!("Grain data len: {:?}", grain_data.payload.len()); flow_reader.destroy().unwrap(); unsafe { mxl_instance.destroy() }.unwrap(); From 666a138ac17f18f9b55f85a003142822ddfe9c26 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Fri, 11 Jul 2025 14:09:03 +0200 Subject: [PATCH 21/77] Reflect function name changes in MXL Signed-off-by: Pavel Cernohorsky --- rust/mxl/examples/flow-reader.rs | 4 ++-- rust/mxl/src/api.rs | 16 ++++++++-------- rust/mxl/src/instance.rs | 4 ++-- rust/mxl/tests/basic_tests.rs | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/rust/mxl/examples/flow-reader.rs b/rust/mxl/examples/flow-reader.rs index 3f7ce597a..f20cb431c 100644 --- a/rust/mxl/examples/flow-reader.rs +++ b/rust/mxl/examples/flow-reader.rs @@ -27,11 +27,11 @@ fn main() -> Result<(), mxl::Error> { let reader = mxl_instance.create_flow_reader(&opts.flow_id)?; let flow_info = reader.get_info()?; let rate = flow_info.discrete_flow_info()?.grainRate; - let current_head_index = mxl_instance.get_current_head_index(&rate); + let current_index = mxl_instance.get_current_index(&rate); info!("Grain rate: {}/{}", rate.numerator, rate.denominator); - for index in current_head_index.. { + for index in current_index.. { let grain_data = reader.get_complete_grain(index, READ_TIMEOUT)?; info!( "Index: {index} Grain data len: {:?}", diff --git a/rust/mxl/src/api.rs b/rust/mxl/src/api.rs index f4941ba30..5cf81e9f0 100644 --- a/rust/mxl/src/api.rs +++ b/rust/mxl/src/api.rs @@ -121,19 +121,19 @@ pub struct MxlApi { mxl_flow_writer_commit_samples: unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, #[allow(non_snake_case)] - #[dlopen2_name = "mxlGetCurrentHeadIndex"] - mxl_get_current_head_index: unsafe extern "C" fn(editRate: *const mxl_sys::Rational) -> u64, + #[dlopen2_name = "mxlGetCurrentIndex"] + mxl_get_current_index: unsafe extern "C" fn(editRate: *const mxl_sys::Rational) -> u64, #[allow(non_snake_case)] - #[dlopen2_name = "mxlGetNsUntilHeadIndex"] - mxl_get_ns_until_head_index: + #[dlopen2_name = "mxlGetNsUntilIndex"] + mxl_get_ns_until_index: unsafe extern "C" fn(index: u64, editRate: *const mxl_sys::Rational) -> u64, #[allow(non_snake_case)] - #[dlopen2_name = "mxlTimestampToHeadIndex"] - mxl_timestamp_to_head_index: + #[dlopen2_name = "mxlTimestampToIndex"] + mxl_timestamp_to_index: unsafe extern "C" fn(editRate: *const mxl_sys::Rational, timestamp: u64) -> u64, #[allow(non_snake_case)] - #[dlopen2_name = "mxlHeadIndexToTimestamp"] - mxl_head_index_to_timestamp: + #[dlopen2_name = "mxlIndexToTimestamp"] + mxl_index_to_timestamp: unsafe extern "C" fn(editRate: *const mxl_sys::Rational, index: u64) -> u64, #[dlopen2_name = "mxlSleepForNs"] mxl_sleep_for_ns: unsafe extern "C" fn(ns: u64), diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index 07a9e1091..bb62e3716 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -78,8 +78,8 @@ impl MxlInstance { Ok(MxlFlowReader::new(self.context.clone(), reader)) } - pub fn get_current_head_index(&self, rational: &mxl_sys::Rational) -> u64 { - unsafe { self.context.api.mxl_get_current_head_index(rational) } + pub fn get_current_index(&self, rational: &mxl_sys::Rational) -> u64 { + unsafe { self.context.api.mxl_get_current_index(rational) } } /// This function forces the destruction of the MXL instance. diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index 7a1ba8560..b8a560ddb 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -38,9 +38,9 @@ fn basic_mxl_writing_reading() { .unwrap(); let flow_info = flow_reader.get_info().unwrap(); let rate = flow_info.discrete_flow_info().unwrap().grainRate; - let current_head_index = mxl_instance.get_current_head_index(&rate); + let current_index = mxl_instance.get_current_index(&rate); let grain_data = flow_reader - .get_complete_grain(current_head_index, Duration::from_secs(5)) + .get_complete_grain(current_index, Duration::from_secs(5)) .unwrap(); let grain_data: OwnedGrainData = grain_data.into(); info!("Grain data len: {:?}", grain_data.payload.len()); From a7cd8cb180ba22b138b33f5c22372fcff8aead30 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Wed, 9 Jul 2025 12:15:23 +0200 Subject: [PATCH 22/77] Add basic discrete flow writer Rust wrapper Signed-off-by: Pavel Cernohorsky --- rust/Cargo.lock | 91 ++++++++++++++++++++++++++ rust/Cargo.toml | 1 + rust/mxl/Cargo.toml | 1 + rust/mxl/build.rs | 4 +- rust/mxl/examples/common/mod.rs | 9 +++ rust/mxl/examples/flow-reader.rs | 16 ++--- rust/mxl/examples/flow-writer.rs | 76 ++++++++++++++++++++++ rust/mxl/src/config.rs | 4 ++ rust/mxl/src/flow.rs | 14 ++++ rust/mxl/src/flow_reader.rs | 14 ++-- rust/mxl/src/flow_writer.rs | 75 ++++++++++++++++++++++ rust/mxl/src/grain_writer.rs | 97 ++++++++++++++++++++++++++++ rust/mxl/src/instance.rs | 107 ++++++++++++++++++++++++++++++- rust/mxl/src/lib.rs | 5 ++ rust/mxl/src/tools.rs | 8 +++ rust/mxl/tests/basic_tests.rs | 40 +++++++++--- 16 files changed, 531 insertions(+), 31 deletions(-) create mode 100644 rust/mxl/examples/common/mod.rs create mode 100644 rust/mxl/examples/flow-writer.rs create mode 100644 rust/mxl/src/flow_writer.rs create mode 100644 rust/mxl/src/grain_writer.rs create mode 100644 rust/mxl/src/tools.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 1423a7750..6a731e570 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -54,6 +54,12 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + [[package]] name = "cexpr" version = "0.6.0" @@ -168,6 +174,16 @@ dependencies = [ "either", ] +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -227,6 +243,7 @@ dependencies = [ "thiserror", "tracing", "tracing-subscriber", + "uuid", ] [[package]] @@ -352,6 +369,12 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + [[package]] name = "sharded-slab" version = "0.1.7" @@ -487,12 +510,80 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +[[package]] +name = "uuid" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 275c009ba..65ca6f082 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -18,6 +18,7 @@ futures = "0.3" thiserror = "2.0.12" tracing = { version = "0.1", features = ["log"] } tracing-subscriber = { version = "0.3", features = ["env-filter", "std"] } +uuid = { version = "1.17" } [workspace.dependencies.clap] version = "4.1.4" diff --git a/rust/mxl/Cargo.toml b/rust/mxl/Cargo.toml index 06f9c3528..509a57899 100644 --- a/rust/mxl/Cargo.toml +++ b/rust/mxl/Cargo.toml @@ -10,6 +10,7 @@ mxl-sys = { path = "../mxl-sys" } dlopen2.workspace = true thiserror.workspace = true tracing.workspace = true +uuid.workspace = true [dev-dependencies] clap.workspace = true diff --git a/rust/mxl/build.rs b/rust/mxl/build.rs index 3ca88a2bd..382d75e18 100644 --- a/rust/mxl/build.rs +++ b/rust/mxl/build.rs @@ -16,7 +16,9 @@ fn main() { .join("constants.rs"); let data = format!( - "pub const MXL_BUILD_DIR: &str = \"{}\";\n", + "pub const MXL_REPO_ROOT: &str = \"{}\";\n\ + pub const MXL_BUILD_DIR: &str = \"{}\";\n", + repo_root.to_string_lossy(), build_dir.to_string_lossy() ); std::fs::write(out_path, data).expect("Unable to write file"); diff --git a/rust/mxl/examples/common/mod.rs b/rust/mxl/examples/common/mod.rs new file mode 100644 index 000000000..221a46291 --- /dev/null +++ b/rust/mxl/examples/common/mod.rs @@ -0,0 +1,9 @@ +pub fn setup_logging() { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::builder() + .with_default_directive(tracing::level_filters::LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .init(); +} diff --git a/rust/mxl/examples/flow-reader.rs b/rust/mxl/examples/flow-reader.rs index f20cb431c..1b222b0d3 100644 --- a/rust/mxl/examples/flow-reader.rs +++ b/rust/mxl/examples/flow-reader.rs @@ -1,3 +1,5 @@ +mod common; + use std::time::Duration; use clap::Parser; @@ -13,13 +15,13 @@ pub struct Opts { #[arg(long)] pub mxl_domain: String, - /// The id of the flow. + /// The id of the flow to read. #[arg(long)] pub flow_id: String, } fn main() -> Result<(), mxl::Error> { - setup_logging(); + common::setup_logging(); let opts: Opts = Opts::parse(); let mxl_api = mxl::load_api(get_mxf_so_path())?; @@ -41,13 +43,3 @@ fn main() -> Result<(), mxl::Error> { Ok(()) } - -fn setup_logging() { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::builder() - .with_default_directive(tracing::level_filters::LevelFilter::INFO.into()) - .from_env_lossy(), - ) - .init(); -} diff --git a/rust/mxl/examples/flow-writer.rs b/rust/mxl/examples/flow-writer.rs new file mode 100644 index 000000000..5c1fb8d53 --- /dev/null +++ b/rust/mxl/examples/flow-writer.rs @@ -0,0 +1,76 @@ +mod common; + +use clap::Parser; +use mxl::config::get_mxf_so_path; +use tracing::info; + +#[derive(Debug, Parser)] +#[command(version = clap::crate_version!(), author = clap::crate_authors!())] +pub struct Opts { + /// The path to the shmem directory where the mxl domain is mapped. + #[arg(long)] + pub mxl_domain: String, + + /// The path to the configuration file describing the flow we want to write to. + #[arg(long)] + pub flow_config_file: String, + + /// The number of grains to write. + #[arg(long)] + pub grain_count: Option, +} + +fn main() -> Result<(), mxl::Error> { + common::setup_logging(); + let opts: Opts = Opts::parse(); + + let mxl_api = mxl::load_api(get_mxf_so_path())?; + let mxl_instance = mxl::MxlInstance::new(mxl_api, &opts.mxl_domain, "")?; + let flow_def = mxl::tools::read_file(opts.flow_config_file.as_str()).map_err(|error| { + mxl::Error::Other(format!( + "Error while reading flow definition from \"{}\": {}", + &opts.flow_config_file, error + )) + })?; + let flow_info = mxl_instance.create_flow(flow_def.as_str(), None)?; + let flow_id = flow_info.common_flow_info().id().to_string(); + let grain_rate = flow_info.discrete_flow_info()?.grainRate; + let mut grain_index = mxl_instance.get_current_index(&grain_rate); + info!( + "Will write to flow \"{flow_id}\" with grain rate {}/{} starting from index {grain_index}.", + grain_rate.numerator, grain_rate.denominator + ); + let writer = mxl_instance.create_flow_writer(flow_id.as_str())?; + + let mut remaining_grains = opts.grain_count; + loop { + if let Some(count) = remaining_grains { + if count == 0 { + break; + } + remaining_grains = Some(count - 1); + } + + let mut grain_writer = writer.open_grain(grain_index)?; + let payload = grain_writer.payload_mut(); + let payload_len = payload.len(); + for i in 0..payload_len { + payload[i] = ((i as u64 + grain_index) % 256) as u8; + } + grain_writer.commit(payload_len as u32)?; + + let timestamp = mxl_instance.index_to_timestamp(grain_index + 1, &grain_rate)?; + let sleep_duration = mxl_instance.get_duration_until_index(grain_index + 1, &grain_rate)?; + info!( + "Finished writing {payload_len} bytes into grain {grain_index}, will sleep for {:?} until timestamp {timestamp}.", + sleep_duration + ); + grain_index += 1; + mxl_instance.sleep_for(sleep_duration); + } + + info!("Finished writing requested number of grains, deleting the flow."); + writer.destroy()?; + mxl_instance.destroy_flow(flow_id.as_str())?; + Ok(()) +} diff --git a/rust/mxl/src/config.rs b/rust/mxl/src/config.rs index 33e2d19b2..f89891ae5 100644 --- a/rust/mxl/src/config.rs +++ b/rust/mxl/src/config.rs @@ -8,3 +8,7 @@ pub fn get_mxf_so_path() -> std::path::PathBuf { .join("lib") .join("libmxl.so") } + +pub fn get_mxl_repo_root() -> std::path::PathBuf { + std::path::PathBuf::from_str(MXL_REPO_ROOT).expect("build error: 'MXL_REPO_ROOT' is invalid") +} diff --git a/rust/mxl/src/flow.rs b/rust/mxl/src/flow.rs index 7b293f629..2bb55ab86 100644 --- a/rust/mxl/src/flow.rs +++ b/rust/mxl/src/flow.rs @@ -1,3 +1,5 @@ +use uuid::Uuid; + use crate::{Error, Result}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -40,4 +42,16 @@ impl FlowInfo { } Ok(unsafe { &self.value.__bindgen_anon_1.discrete }) } + + pub fn common_flow_info(&self) -> CommonFlowInfo { + CommonFlowInfo(&self.value.common) + } +} + +pub struct CommonFlowInfo<'a>(&'a mxl_sys::CommonFlowInfo); + +impl CommonFlowInfo<'_> { + pub fn id(&self) -> Uuid { + Uuid::from_bytes(self.0.id) + } } diff --git a/rust/mxl/src/flow_reader.rs b/rust/mxl/src/flow_reader.rs index 51d1d2d52..66b16b362 100644 --- a/rust/mxl/src/flow_reader.rs +++ b/rust/mxl/src/flow_reader.rs @@ -79,24 +79,20 @@ impl MxlFlowReader { let mut reader = std::ptr::null_mut(); std::mem::swap(&mut self.reader, &mut reader); - let result = Error::from_status(unsafe { + Error::from_status(unsafe { self.context .api .mxl_release_flow_reader(self.context.instance, reader) - }); - - if let Err(err) = &result { - tracing::error!("Failed to release MXL flow reader: {:?}", err); - } - - result + }) } } impl Drop for MxlFlowReader { fn drop(&mut self) { if !self.reader.is_null() { - let _ = self.destroy_inner(); + if let Err(err) = self.destroy_inner() { + tracing::error!("Failed to release MXL flow reader: {:?}", err); + } } } } diff --git a/rust/mxl/src/flow_writer.rs b/rust/mxl/src/flow_writer.rs new file mode 100644 index 000000000..13aa0d126 --- /dev/null +++ b/rust/mxl/src/flow_writer.rs @@ -0,0 +1,75 @@ +use std::sync::Arc; + +use crate::{Error, Result, grain_writer::GrainWriter, instance::InstanceContext}; + +/// MXL Flow Writer for discrete flows (grain-based data like video frames) +pub struct MxlFlowWriter { + context: Arc, + writer: mxl_sys::mxlFlowWriter, +} + +impl MxlFlowWriter { + pub(crate) fn new(context: Arc, writer: mxl_sys::mxlFlowWriter) -> Self { + Self { context, writer } + } + + pub fn destroy(mut self) -> Result<()> { + self.destroy_inner() + } + + /// The current MXL implementation states a TODO to allow multiple grains to be edited at the + /// same time. For this reason, there is no protection on the Rust level against trying to open + /// multiple grains. If the TODO ever gets removed, it may be worth considering pattern where + /// opening grain would consume the writer and then return it back on commit or cancel. + pub fn open_grain<'a>(&'a self, index: u64) -> Result> { + let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; + let mut payload_ptr: *mut u8 = std::ptr::null_mut(); + unsafe { + Error::from_status(self.context.api.mxl_flow_writer_open_grain( + self.writer, + index, + &mut grain_info, + &mut payload_ptr, + ))?; + } + + if payload_ptr.is_null() { + return Err(Error::Other(format!( + "Failed to open grain payload for index {}.", + index + ))); + } + + Ok(GrainWriter::new( + self.context.clone(), + self.writer, + grain_info, + payload_ptr, + )) + } + + fn destroy_inner(&mut self) -> Result<()> { + if self.writer.is_null() { + return Err(Error::InvalidArg); + } + + let mut writer = std::ptr::null_mut(); + std::mem::swap(&mut self.writer, &mut writer); + + Error::from_status(unsafe { + self.context + .api + .mxl_release_flow_writer(self.context.instance, writer) + }) + } +} + +impl Drop for MxlFlowWriter { + fn drop(&mut self) { + if !self.writer.is_null() { + if let Err(err) = self.destroy_inner() { + tracing::error!("Failed to release MXL flow writer: {:?}", err); + } + } + } +} diff --git a/rust/mxl/src/grain_writer.rs b/rust/mxl/src/grain_writer.rs new file mode 100644 index 000000000..a5932f9a3 --- /dev/null +++ b/rust/mxl/src/grain_writer.rs @@ -0,0 +1,97 @@ +use std::marker::PhantomData; +use std::sync::Arc; + +use tracing::error; + +use crate::{Error, Result, instance::InstanceContext}; + +/// RAII grain writing session +/// +/// Automatically cancels the grain if not explicitly committed. +pub struct GrainWriter<'a> { + context: Arc, + writer: mxl_sys::mxlFlowWriter, + grain_info: mxl_sys::GrainInfo, + payload_ptr: *mut u8, + /// Serves as a flag to know whether to cancel the grain on drop. + committed_or_canceled: bool, + phantom: PhantomData<&'a ()>, +} + +impl<'a> GrainWriter<'a> { + pub(crate) fn new( + context: Arc, + writer: mxl_sys::mxlFlowWriter, + grain_info: mxl_sys::GrainInfo, + payload_ptr: *mut u8, + ) -> Self { + Self { + context, + writer, + grain_info, + payload_ptr, + committed_or_canceled: false, + phantom: Default::default(), + } + } + + pub fn payload_mut(&mut self) -> &mut [u8] { + unsafe { + std::slice::from_raw_parts_mut(self.payload_ptr, self.grain_info.grainSize as usize) + } + } + + pub fn user_data_mut(&mut self) -> &mut [u8] { + &mut self.grain_info.userData + } + + pub fn max_size(&self) -> u32 { + self.grain_info.grainSize + } + + pub fn committed_size(&self) -> u32 { + self.grain_info.commitedSize + } + + pub fn commit(mut self, commited_size: u32) -> Result<()> { + self.committed_or_canceled = true; + + if commited_size > self.grain_info.grainSize { + return Err(Error::Other(format!( + "Commited size {} cannot exceed grain size {}.", + commited_size, self.grain_info.grainSize + ))); + } + self.grain_info.commitedSize = commited_size; + + unsafe { + Error::from_status( + self.context + .api + .mxl_flow_writer_commit_grain(self.writer, &self.grain_info), + ) + } + } + + /// Please note that the behavior of canceling a grain writing is dependent on the behavior + /// implemented in MXL itself. Particularly, if grain data has been mutated and then writing + /// canceled, mutation will most likely stay in place, only head won't be updated, and readers + /// notified. + pub fn cancel(mut self) -> Result<()> { + self.committed_or_canceled = true; + + unsafe { Error::from_status(self.context.api.mxl_flow_writer_cancel_grain(self.writer)) } + } +} + +impl<'a> Drop for GrainWriter<'a> { + fn drop(&mut self) { + if !self.committed_or_canceled { + if let Err(error) = unsafe { + Error::from_status(self.context.api.mxl_flow_writer_cancel_grain(self.writer)) + } { + error!("Failed to cancel grain write on drop: {:?}", error); + } + } + } +} diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index bb62e3716..4c91aa0a8 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -2,7 +2,7 @@ use std::{ffi::CString, sync::Arc}; use dlopen2::wrapper::Container; -use crate::{Error, MxlApi, MxlFlowReader, Result}; +use crate::{Error, FlowInfo, MxlApi, MxlFlowReader, MxlFlowWriter, Result}; /// This struct stores the context that is shared by all objects. /// It is separated out from `MxlInstance` so that it can be cloned @@ -78,10 +78,115 @@ impl MxlInstance { Ok(MxlFlowReader::new(self.context.clone(), reader)) } + pub fn create_flow_writer(&self, flow_id: &str) -> Result { + let flow_id = CString::new(flow_id)?; + let options = CString::new("")?; + let mut writer: mxl_sys::mxlFlowWriter = std::ptr::null_mut(); + unsafe { + Error::from_status(self.context.api.mxl_create_flow_writer( + self.context.instance, + flow_id.as_ptr(), + options.as_ptr(), + &mut writer, + ))?; + } + if writer.is_null() { + return Err(Error::Other("Failed to create flow writer.".to_string())); + } + Ok(MxlFlowWriter::new(self.context.clone(), writer)) + } + + /// For now, we provide direct access to the MXL API for creating and + /// destroying flows. Maybe it would be worth to provide RAII wrapper... + /// Instead? As well? + pub fn create_flow(&self, flow_def: &str, options: Option<&str>) -> Result { + let flow_def = CString::new(flow_def)?; + let options = CString::new(options.unwrap_or(""))?; + let mut info = std::mem::MaybeUninit::::uninit(); + + unsafe { + Error::from_status(self.context.api.mxl_create_flow( + self.context.instance, + flow_def.as_ptr(), + options.as_ptr(), + info.as_mut_ptr(), + ))?; + } + + let info = unsafe { info.assume_init() }; + Ok(FlowInfo { value: info }) + } + + /// See `create_flow` for more info. + pub fn destroy_flow(&self, flow_id: &str) -> Result<()> { + let flow_id = CString::new(flow_id)?; + unsafe { + Error::from_status( + self.context + .api + .mxl_destroy_flow(self.context.instance, flow_id.as_ptr()), + )?; + } + Ok(()) + } + pub fn get_current_index(&self, rational: &mxl_sys::Rational) -> u64 { unsafe { self.context.api.mxl_get_current_index(rational) } } + pub fn get_duration_until_index( + &self, + index: u64, + rate: &mxl_sys::Rational, + ) -> Result { + let duration_ns = unsafe { self.context.api.mxl_get_ns_until_index(index, rate) }; + if duration_ns == u64::MAX { + Err(Error::Other(format!( + "Failed to get duration until index, invalid rate {}/{}.", + rate.numerator, rate.denominator + ))) + } else { + Ok(std::time::Duration::from_nanos(duration_ns)) + } + } + + /// TODO: Make timestamp a strong type. + pub fn timestamp_to_index(&self, timestamp: u64, rate: &mxl_sys::Rational) -> Result { + let index = unsafe { self.context.api.mxl_timestamp_to_index(rate, timestamp) }; + if index == u64::MAX { + Err(Error::Other(format!( + "Failed to convert timestamp to index, invalid rate {}/{}.", + rate.numerator, rate.denominator + ))) + } else { + Ok(index) + } + } + + pub fn index_to_timestamp(&self, index: u64, rate: &mxl_sys::Rational) -> Result { + let timestamp = unsafe { self.context.api.mxl_index_to_timestamp(rate, index) }; + if timestamp == u64::MAX { + Err(Error::Other(format!( + "Failed to convert index to timestamp, invalid rate {}/{}.", + rate.numerator, rate.denominator + ))) + } else { + Ok(timestamp) + } + } + + pub fn sleep_for(&self, duration: std::time::Duration) { + unsafe { + self.context + .api + .mxl_sleep_for_ns(duration.as_nanos() as u64) + } + } + + pub fn get_time(&self) -> u64 { + unsafe { self.context.api.mxl_get_time() } + } + /// This function forces the destruction of the MXL instance. /// It is not safe as other objects may still be using it. /// It is meant for testing purposes only. diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index 3c388c6dd..ce0f623ba 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -2,14 +2,19 @@ mod api; mod error; mod flow; mod flow_reader; +mod flow_writer; mod grain_data; +mod grain_writer; mod instance; pub mod config; +pub mod tools; pub use api::{MxlApi, load_api}; pub use error::{Error, Result}; pub use flow::*; pub use flow_reader::MxlFlowReader; +pub use flow_writer::MxlFlowWriter; pub use grain_data::*; +pub use grain_writer::GrainWriter; pub use instance::MxlInstance; diff --git a/rust/mxl/src/tools.rs b/rust/mxl/src/tools.rs new file mode 100644 index 000000000..312af2180 --- /dev/null +++ b/rust/mxl/src/tools.rs @@ -0,0 +1,8 @@ +pub fn read_file(file_path: impl AsRef) -> Result { + use std::io::Read; + + let mut file = std::fs::File::open(file_path)?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + Ok(contents) +} diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index b8a560ddb..7a34402f7 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -9,6 +9,18 @@ use tracing::info; static LOG_ONCE: std::sync::Once = std::sync::Once::new(); +fn setup_empty_domain() -> String { + // TODO: Randomize the domain name to allow parallel tests run? + // On the other hand, some tests may leave stuff behind, this way we do not keep + // increasing garbage from the tests. + let result = "/dev/shm/mxl_rust_unit_tests_domain"; + if std::path::Path::new(result).exists() { + std::fs::remove_dir_all(result).expect("Failed to remove existing test domain directory"); + } + std::fs::create_dir_all(result).expect("Failed to create test domain directory"); + result.to_string() +} + fn setup_test() -> mxl::MxlInstance { // Set up the logging to use the RUST_LOG environment variable and if not present, print INFO // and higher. @@ -23,27 +35,39 @@ fn setup_test() -> mxl::MxlInstance { }); let mxl_api = mxl::load_api(get_mxf_so_path()).unwrap(); - // TODO: Randomize the domain name to allow parallel tests run. - mxl::MxlInstance::new(mxl_api, "/dev/shm/mxl_domain", "").unwrap() + let domain = setup_empty_domain(); + mxl::MxlInstance::new(mxl_api, &domain, "").unwrap() } #[test] fn basic_mxl_writing_reading() { - // TODO: Add the writing part of the test. Now the test requires the external MXL tool to write - // data: `tools/mxl-gst/mxl-gst-videotestsrc -d /dev/shm/mxl_domain -f - // ../../lib/tests/data/v210_flow.json` let mxl_instance = setup_test(); - let flow_reader = mxl_instance - .create_flow_reader("5fbec3b1-1b0f-417d-9059-8b94a47197ed") + let flow_config_file = mxl::config::get_mxl_repo_root().join("lib/tests/data/v210_flow.json"); + let flow_def = mxl::tools::read_file(flow_config_file.as_path()) + .map_err(|error| { + mxl::Error::Other(format!( + "Error while reading flow definition from \"{}\": {}", + flow_config_file.display(), + error + )) + }) .unwrap(); - let flow_info = flow_reader.get_info().unwrap(); + let flow_info = mxl_instance.create_flow(flow_def.as_str(), None).unwrap(); + let flow_id = flow_info.common_flow_info().id().to_string(); + let flow_writer = mxl_instance.create_flow_writer(flow_id.as_str()).unwrap(); + let flow_reader = mxl_instance.create_flow_reader(flow_id.as_str()).unwrap(); let rate = flow_info.discrete_flow_info().unwrap().grainRate; let current_index = mxl_instance.get_current_index(&rate); + let grain_writer = flow_writer.open_grain(current_index).unwrap(); + let grain_size = grain_writer.max_size(); + grain_writer.commit(grain_size).unwrap(); let grain_data = flow_reader .get_complete_grain(current_index, Duration::from_secs(5)) .unwrap(); let grain_data: OwnedGrainData = grain_data.into(); info!("Grain data len: {:?}", grain_data.payload.len()); flow_reader.destroy().unwrap(); + flow_writer.destroy().unwrap(); + mxl_instance.destroy_flow(flow_id.as_str()).unwrap(); unsafe { mxl_instance.destroy() }.unwrap(); } From 017d0b66ab9607c157f7f275e3d5cdfe335f3bad Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Wed, 9 Jul 2025 12:58:11 +0200 Subject: [PATCH 23/77] Fix double free on explicit instance destruction in test - Our unsafe code was way too unsafe :) Signed-off-by: Pavel Cernohorsky --- rust/mxl/src/instance.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index 4c91aa0a8..19954f2b0 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -26,8 +26,12 @@ impl InstanceContext { /// /// The caller must ensure that no other objects are using the MXL instance when this function is called. /// Calling this function while other references exist may lead to undefined behavior. - pub unsafe fn destroy(&self) -> Result<()> { - unsafe { self.api.mxl_destroy_instance(self.instance) }; + pub unsafe fn destroy(&mut self) -> Result<()> { + unsafe { + let mut instance = std::ptr::null_mut(); + std::mem::swap(&mut self.instance, &mut instance); + self.api.mxl_destroy_instance(self.instance) + }; Ok(()) } } @@ -194,8 +198,11 @@ impl MxlInstance { /// # Safety /// /// The caller must ensure that no other objects are using the MXL instance when this function is called. - /// Calling this function while other references exist may lead to undefined behavior. + /// Calling this function while other references exist will lead to panic. pub unsafe fn destroy(self) -> Result<()> { - unsafe { self.context.destroy() } + unsafe { + let mut context = Arc::into_inner(self.context).unwrap(); + context.destroy() + } } } From 9ad5676281972f9f1056a2303f4aaf5c4fa58a21 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Thu, 10 Jul 2025 10:58:38 +0200 Subject: [PATCH 24/77] Rename GrainWriter to GrainWriteAccess - Need to free up the name GrainWriter for future chages - there will be a GrainWriter and a SamplesWriter, analogically to how MXL distinguishes access to "discrete" and "continuous" flows. Signed-off-by: Pavel Cernohorsky --- rust/mxl/examples/flow-writer.rs | 6 +++--- rust/mxl/src/flow_writer.rs | 6 +++--- rust/mxl/src/{grain_writer.rs => grain_write_access.rs} | 6 +++--- rust/mxl/src/lib.rs | 4 ++-- rust/mxl/tests/basic_tests.rs | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) rename rust/mxl/src/{grain_writer.rs => grain_write_access.rs} (96%) diff --git a/rust/mxl/examples/flow-writer.rs b/rust/mxl/examples/flow-writer.rs index 5c1fb8d53..72044fd7e 100644 --- a/rust/mxl/examples/flow-writer.rs +++ b/rust/mxl/examples/flow-writer.rs @@ -51,13 +51,13 @@ fn main() -> Result<(), mxl::Error> { remaining_grains = Some(count - 1); } - let mut grain_writer = writer.open_grain(grain_index)?; - let payload = grain_writer.payload_mut(); + let mut grain_writer_access = writer.open_grain(grain_index)?; + let payload = grain_writer_access.payload_mut(); let payload_len = payload.len(); for i in 0..payload_len { payload[i] = ((i as u64 + grain_index) % 256) as u8; } - grain_writer.commit(payload_len as u32)?; + grain_writer_access.commit(payload_len as u32)?; let timestamp = mxl_instance.index_to_timestamp(grain_index + 1, &grain_rate)?; let sleep_duration = mxl_instance.get_duration_until_index(grain_index + 1, &grain_rate)?; diff --git a/rust/mxl/src/flow_writer.rs b/rust/mxl/src/flow_writer.rs index 13aa0d126..0bb29f80b 100644 --- a/rust/mxl/src/flow_writer.rs +++ b/rust/mxl/src/flow_writer.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use crate::{Error, Result, grain_writer::GrainWriter, instance::InstanceContext}; +use crate::{Error, Result, grain_write_access::GrainWriteAccess, instance::InstanceContext}; /// MXL Flow Writer for discrete flows (grain-based data like video frames) pub struct MxlFlowWriter { @@ -21,7 +21,7 @@ impl MxlFlowWriter { /// same time. For this reason, there is no protection on the Rust level against trying to open /// multiple grains. If the TODO ever gets removed, it may be worth considering pattern where /// opening grain would consume the writer and then return it back on commit or cancel. - pub fn open_grain<'a>(&'a self, index: u64) -> Result> { + pub fn open_grain<'a>(&'a self, index: u64) -> Result> { let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; let mut payload_ptr: *mut u8 = std::ptr::null_mut(); unsafe { @@ -40,7 +40,7 @@ impl MxlFlowWriter { ))); } - Ok(GrainWriter::new( + Ok(GrainWriteAccess::new( self.context.clone(), self.writer, grain_info, diff --git a/rust/mxl/src/grain_writer.rs b/rust/mxl/src/grain_write_access.rs similarity index 96% rename from rust/mxl/src/grain_writer.rs rename to rust/mxl/src/grain_write_access.rs index a5932f9a3..6396bb169 100644 --- a/rust/mxl/src/grain_writer.rs +++ b/rust/mxl/src/grain_write_access.rs @@ -8,7 +8,7 @@ use crate::{Error, Result, instance::InstanceContext}; /// RAII grain writing session /// /// Automatically cancels the grain if not explicitly committed. -pub struct GrainWriter<'a> { +pub struct GrainWriteAccess<'a> { context: Arc, writer: mxl_sys::mxlFlowWriter, grain_info: mxl_sys::GrainInfo, @@ -18,7 +18,7 @@ pub struct GrainWriter<'a> { phantom: PhantomData<&'a ()>, } -impl<'a> GrainWriter<'a> { +impl<'a> GrainWriteAccess<'a> { pub(crate) fn new( context: Arc, writer: mxl_sys::mxlFlowWriter, @@ -84,7 +84,7 @@ impl<'a> GrainWriter<'a> { } } -impl<'a> Drop for GrainWriter<'a> { +impl<'a> Drop for GrainWriteAccess<'a> { fn drop(&mut self) { if !self.committed_or_canceled { if let Err(error) = unsafe { diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index ce0f623ba..4eb64e946 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -4,7 +4,7 @@ mod flow; mod flow_reader; mod flow_writer; mod grain_data; -mod grain_writer; +mod grain_write_access; mod instance; pub mod config; @@ -16,5 +16,5 @@ pub use flow::*; pub use flow_reader::MxlFlowReader; pub use flow_writer::MxlFlowWriter; pub use grain_data::*; -pub use grain_writer::GrainWriter; +pub use grain_write_access::GrainWriteAccess; pub use instance::MxlInstance; diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index 7a34402f7..f47e6ead4 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -58,9 +58,9 @@ fn basic_mxl_writing_reading() { let flow_reader = mxl_instance.create_flow_reader(flow_id.as_str()).unwrap(); let rate = flow_info.discrete_flow_info().unwrap().grainRate; let current_index = mxl_instance.get_current_index(&rate); - let grain_writer = flow_writer.open_grain(current_index).unwrap(); - let grain_size = grain_writer.max_size(); - grain_writer.commit(grain_size).unwrap(); + let grain_write_access = flow_writer.open_grain(current_index).unwrap(); + let grain_size = grain_write_access.max_size(); + grain_write_access.commit(grain_size).unwrap(); let grain_data = flow_reader .get_complete_grain(current_index, Duration::from_secs(5)) .unwrap(); From 69a2bd5bc707b4866cfb04171ee707d74aa5c0cb Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Thu, 10 Jul 2025 13:30:04 +0200 Subject: [PATCH 25/77] Change MxlFlowWriter into a factory class for more specific writers - Added grain writer and a skeleton for samples writer. Signed-off-by: Pavel Cernohorsky --- rust/mxl/examples/flow-writer.rs | 4 +- rust/mxl/src/flow.rs | 19 +++--- rust/mxl/src/flow_writer.rs | 102 +++++++++++++++++-------------- rust/mxl/src/grain_writer.rs | 75 +++++++++++++++++++++++ rust/mxl/src/instance.rs | 41 ++++++++----- rust/mxl/src/lib.rs | 2 + rust/mxl/src/samples_writer.rs | 44 +++++++++++++ rust/mxl/tests/basic_tests.rs | 5 +- 8 files changed, 218 insertions(+), 74 deletions(-) create mode 100644 rust/mxl/src/grain_writer.rs create mode 100644 rust/mxl/src/samples_writer.rs diff --git a/rust/mxl/examples/flow-writer.rs b/rust/mxl/examples/flow-writer.rs index 72044fd7e..336a43a27 100644 --- a/rust/mxl/examples/flow-writer.rs +++ b/rust/mxl/examples/flow-writer.rs @@ -40,7 +40,9 @@ fn main() -> Result<(), mxl::Error> { "Will write to flow \"{flow_id}\" with grain rate {}/{} starting from index {grain_index}.", grain_rate.numerator, grain_rate.denominator ); - let writer = mxl_instance.create_flow_writer(flow_id.as_str())?; + let writer = mxl_instance + .create_flow_writer(flow_id.as_str())? + .to_grain_writer()?; let mut remaining_grains = opts.grain_count; loop { diff --git a/rust/mxl/src/flow.rs b/rust/mxl/src/flow.rs index 2bb55ab86..b0c63933e 100644 --- a/rust/mxl/src/flow.rs +++ b/rust/mxl/src/flow.rs @@ -15,26 +15,27 @@ impl From for DataFormat { fn from(value: u32) -> Self { match value { 0 => DataFormat::Unspecified, - 1 => DataFormat::Video, - 2 => DataFormat::Audio, - 3 => DataFormat::Data, - 4 => DataFormat::Mux, + mxl_sys::MXL_DATA_FORMAT_VIDEO => DataFormat::Video, + mxl_sys::MXL_DATA_FORMAT_AUDIO => DataFormat::Audio, + mxl_sys::MXL_DATA_FORMAT_DATA => DataFormat::Data, + mxl_sys::MXL_DATA_FORMAT_MUX => DataFormat::Mux, _ => DataFormat::Unspecified, } } } +pub(crate) fn is_discrete_data_format(format: u32) -> bool { + // Check is based on mxlIsDiscreteDataFormat, which is inline, thus not accessible in mxl_sys. + format == mxl_sys::MXL_DATA_FORMAT_VIDEO || format == mxl_sys::MXL_DATA_FORMAT_DATA +} + pub struct FlowInfo { pub(crate) value: mxl_sys::FlowInfo, } impl FlowInfo { pub fn discrete_flow_info(&self) -> Result<&mxl_sys::DiscreteFlowInfo> { - // Check is based on mxlIsDiscreteDataFormat, which is inline, thus not accessible in - // mxl_sys. - if self.value.common.format != mxl_sys::MXL_DATA_FORMAT_VIDEO - && self.value.common.format != mxl_sys::MXL_DATA_FORMAT_DATA - { + if !is_discrete_data_format(self.value.common.format) { return Err(Error::Other(format!( "Flow format is {}, video or data required.", self.value.common.format diff --git a/rust/mxl/src/flow_writer.rs b/rust/mxl/src/flow_writer.rs index 0bb29f80b..ab677fcc9 100644 --- a/rust/mxl/src/flow_writer.rs +++ b/rust/mxl/src/flow_writer.rs @@ -1,73 +1,83 @@ use std::sync::Arc; -use crate::{Error, Result, grain_write_access::GrainWriteAccess, instance::InstanceContext}; +use crate::flow::is_discrete_data_format; +use crate::grain_writer::GrainWriter; +use crate::instance::create_flow_reader; +use crate::samples_writer::SamplesWriter; +use crate::{DataFormat, Error, Result, instance::InstanceContext}; -/// MXL Flow Writer for discrete flows (grain-based data like video frames) +/// Generic MXL Flow Writer, which can be further used to build either the "discrete" (grain-based +/// data like video frames or meta) or "continuous" (audio samples) flow writers in MXL terminology. pub struct MxlFlowWriter { context: Arc, writer: mxl_sys::mxlFlowWriter, + id: uuid::Uuid, } impl MxlFlowWriter { - pub(crate) fn new(context: Arc, writer: mxl_sys::mxlFlowWriter) -> Self { - Self { context, writer } - } - - pub fn destroy(mut self) -> Result<()> { - self.destroy_inner() - } - - /// The current MXL implementation states a TODO to allow multiple grains to be edited at the - /// same time. For this reason, there is no protection on the Rust level against trying to open - /// multiple grains. If the TODO ever gets removed, it may be worth considering pattern where - /// opening grain would consume the writer and then return it back on commit or cancel. - pub fn open_grain<'a>(&'a self, index: u64) -> Result> { - let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; - let mut payload_ptr: *mut u8 = std::ptr::null_mut(); - unsafe { - Error::from_status(self.context.api.mxl_flow_writer_open_grain( - self.writer, - index, - &mut grain_info, - &mut payload_ptr, - ))?; + pub(crate) fn new( + context: Arc, + writer: mxl_sys::mxlFlowWriter, + id: uuid::Uuid, + ) -> Self { + Self { + context, + writer, + id, } + } - if payload_ptr.is_null() { + pub fn to_grain_writer(mut self) -> Result { + let flow_type = self.get_flow_type()?; + if !is_discrete_data_format(flow_type) { return Err(Error::Other(format!( - "Failed to open grain payload for index {}.", - index + "Cannot convert MxlFlowWriter to GrainWriter for continuous flow of type \"{:?}\".", + DataFormat::from(flow_type) ))); } - - Ok(GrainWriteAccess::new( - self.context.clone(), - self.writer, - grain_info, - payload_ptr, - )) + let result = GrainWriter::new(self.context.clone(), self.writer); + self.writer = std::ptr::null_mut(); + Ok(result) } - fn destroy_inner(&mut self) -> Result<()> { - if self.writer.is_null() { - return Err(Error::InvalidArg); + pub fn to_samples_writer(mut self) -> Result { + let flow_type = self.get_flow_type()?; + if is_discrete_data_format(flow_type) { + return Err(Error::Other(format!( + "Cannot convert MxlFlowWriter to SamplesWriter for discrete flow of type \"{:?}\".", + DataFormat::from(flow_type) + ))); } + let result = SamplesWriter::new(self.context.clone(), self.writer); + self.writer = std::ptr::null_mut(); + Ok(result) + } - let mut writer = std::ptr::null_mut(); - std::mem::swap(&mut self.writer, &mut writer); - - Error::from_status(unsafe { - self.context - .api - .mxl_release_flow_writer(self.context.instance, writer) - }) + fn get_flow_type(&self) -> Result { + // This feels pretty ugly, but currently, the only way how to get a flow type in MXL is to + // use a reader. + let reader = create_flow_reader(&self.context, &self.id.to_string()).map_err(|error| { + Error::Other(format!( + "Error while creating flow reader to get the flow type: {error}" + )) + })?; + let flow_info = reader.get_info().map_err(|error| { + Error::Other(format!( + "Error while getting flow type from temporary reader: {error}" + )) + })?; + Ok(flow_info.value.common.format) } } impl Drop for MxlFlowWriter { fn drop(&mut self) { if !self.writer.is_null() { - if let Err(err) = self.destroy_inner() { + if let Err(err) = Error::from_status(unsafe { + self.context + .api + .mxl_release_flow_writer(self.context.instance, self.writer) + }) { tracing::error!("Failed to release MXL flow writer: {:?}", err); } } diff --git a/rust/mxl/src/grain_writer.rs b/rust/mxl/src/grain_writer.rs new file mode 100644 index 000000000..504f881af --- /dev/null +++ b/rust/mxl/src/grain_writer.rs @@ -0,0 +1,75 @@ +use std::sync::Arc; + +use crate::{Error, Result, grain_write_access::GrainWriteAccess, instance::InstanceContext}; + +/// MXL Flow Writer for discrete flows (grain-based data like video frames) +pub struct GrainWriter { + context: Arc, + writer: mxl_sys::mxlFlowWriter, +} + +impl GrainWriter { + pub(crate) fn new(context: Arc, writer: mxl_sys::mxlFlowWriter) -> Self { + Self { context, writer } + } + + pub fn destroy(mut self) -> Result<()> { + self.destroy_inner() + } + + /// The current MXL implementation states a TODO to allow multiple grains to be edited at the + /// same time. For this reason, there is no protection on the Rust level against trying to open + /// multiple grains. If the TODO ever gets removed, it may be worth considering pattern where + /// opening grain would consume the writer and then return it back on commit or cancel. + pub fn open_grain<'a>(&'a self, index: u64) -> Result> { + let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; + let mut payload_ptr: *mut u8 = std::ptr::null_mut(); + unsafe { + Error::from_status(self.context.api.mxl_flow_writer_open_grain( + self.writer, + index, + &mut grain_info, + &mut payload_ptr, + ))?; + } + + if payload_ptr.is_null() { + return Err(Error::Other(format!( + "Failed to open grain payload for index {}.", + index + ))); + } + + Ok(GrainWriteAccess::new( + self.context.clone(), + self.writer, + grain_info, + payload_ptr, + )) + } + + fn destroy_inner(&mut self) -> Result<()> { + if self.writer.is_null() { + return Err(Error::InvalidArg); + } + + let mut writer = std::ptr::null_mut(); + std::mem::swap(&mut self.writer, &mut writer); + + Error::from_status(unsafe { + self.context + .api + .mxl_release_flow_writer(self.context.instance, writer) + }) + } +} + +impl Drop for GrainWriter { + fn drop(&mut self) { + if !self.writer.is_null() { + if let Err(err) = self.destroy_inner() { + tracing::error!("Failed to release MXL flow writer (discrete): {:?}", err); + } + } + } +} diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index 19954f2b0..eab1c6ac4 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -44,6 +44,27 @@ impl Drop for InstanceContext { } } +pub(crate) fn create_flow_reader( + context: &Arc, + flow_id: &str, +) -> Result { + let flow_id = CString::new(flow_id)?; + let options = CString::new("")?; + let mut reader: mxl_sys::mxlFlowReader = std::ptr::null_mut(); + unsafe { + Error::from_status(context.api.mxl_create_flow_reader( + context.instance, + flow_id.as_ptr(), + options.as_ptr(), + &mut reader, + ))?; + } + if reader.is_null() { + return Err(Error::Other("Failed to create flow reader.".to_string())); + } + Ok(MxlFlowReader::new(context.clone(), reader)) +} + pub struct MxlInstance { context: Arc, } @@ -65,24 +86,12 @@ impl MxlInstance { } pub fn create_flow_reader(&self, flow_id: &str) -> Result { - let flow_id = CString::new(flow_id)?; - let options = CString::new("")?; - let mut reader: mxl_sys::mxlFlowReader = std::ptr::null_mut(); - unsafe { - Error::from_status(self.context.api.mxl_create_flow_reader( - self.context.instance, - flow_id.as_ptr(), - options.as_ptr(), - &mut reader, - ))?; - } - if reader.is_null() { - return Err(Error::Other("Failed to create flow reader.".to_string())); - } - Ok(MxlFlowReader::new(self.context.clone(), reader)) + create_flow_reader(&self.context, flow_id) } pub fn create_flow_writer(&self, flow_id: &str) -> Result { + let uuid = uuid::Uuid::parse_str(flow_id) + .map_err(|_| Error::Other("Invalid flow ID format.".to_string()))?; let flow_id = CString::new(flow_id)?; let options = CString::new("")?; let mut writer: mxl_sys::mxlFlowWriter = std::ptr::null_mut(); @@ -97,7 +106,7 @@ impl MxlInstance { if writer.is_null() { return Err(Error::Other("Failed to create flow writer.".to_string())); } - Ok(MxlFlowWriter::new(self.context.clone(), writer)) + Ok(MxlFlowWriter::new(self.context.clone(), writer, uuid)) } /// For now, we provide direct access to the MXL API for creating and diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index 4eb64e946..5167a50d6 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -5,7 +5,9 @@ mod flow_reader; mod flow_writer; mod grain_data; mod grain_write_access; +mod grain_writer; mod instance; +mod samples_writer; pub mod config; pub mod tools; diff --git a/rust/mxl/src/samples_writer.rs b/rust/mxl/src/samples_writer.rs new file mode 100644 index 000000000..e14b88a59 --- /dev/null +++ b/rust/mxl/src/samples_writer.rs @@ -0,0 +1,44 @@ +use std::sync::Arc; + +use crate::{Error, Result, instance::InstanceContext}; + +/// MXL Flow Writer for continuous flows (samples-based data like audio) +pub struct SamplesWriter { + context: Arc, + writer: mxl_sys::mxlFlowWriter, +} + +impl SamplesWriter { + pub(crate) fn new(context: Arc, writer: mxl_sys::mxlFlowWriter) -> Self { + Self { context, writer } + } + + pub fn destroy(mut self) -> Result<()> { + self.destroy_inner() + } + + fn destroy_inner(&mut self) -> Result<()> { + if self.writer.is_null() { + return Err(Error::InvalidArg); + } + + let mut writer = std::ptr::null_mut(); + std::mem::swap(&mut self.writer, &mut writer); + + Error::from_status(unsafe { + self.context + .api + .mxl_release_flow_writer(self.context.instance, writer) + }) + } +} + +impl Drop for SamplesWriter { + fn drop(&mut self) { + if !self.writer.is_null() { + if let Err(err) = self.destroy_inner() { + tracing::error!("Failed to release MXL flow writer (continuous): {:?}", err); + } + } + } +} diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index f47e6ead4..762282e20 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -55,10 +55,11 @@ fn basic_mxl_writing_reading() { let flow_info = mxl_instance.create_flow(flow_def.as_str(), None).unwrap(); let flow_id = flow_info.common_flow_info().id().to_string(); let flow_writer = mxl_instance.create_flow_writer(flow_id.as_str()).unwrap(); + let grain_writer = flow_writer.to_grain_writer().unwrap(); let flow_reader = mxl_instance.create_flow_reader(flow_id.as_str()).unwrap(); let rate = flow_info.discrete_flow_info().unwrap().grainRate; let current_index = mxl_instance.get_current_index(&rate); - let grain_write_access = flow_writer.open_grain(current_index).unwrap(); + let grain_write_access = grain_writer.open_grain(current_index).unwrap(); let grain_size = grain_write_access.max_size(); grain_write_access.commit(grain_size).unwrap(); let grain_data = flow_reader @@ -67,7 +68,7 @@ fn basic_mxl_writing_reading() { let grain_data: OwnedGrainData = grain_data.into(); info!("Grain data len: {:?}", grain_data.payload.len()); flow_reader.destroy().unwrap(); - flow_writer.destroy().unwrap(); + grain_writer.destroy().unwrap(); mxl_instance.destroy_flow(flow_id.as_str()).unwrap(); unsafe { mxl_instance.destroy() }.unwrap(); } From b32eb783e081820c635932c1ec85c415ee1c2d14 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Fri, 11 Jul 2025 14:58:00 +0200 Subject: [PATCH 26/77] Add basic continuous flow writer Rust wrapper Signed-off-by: Pavel Cernohorsky --- rust/mxl/examples/flow-writer.rs | 101 +++++++++++++++++++++++++-- rust/mxl/src/flow.rs | 14 ++++ rust/mxl/src/lib.rs | 2 + rust/mxl/src/samples_write_access.rs | 95 +++++++++++++++++++++++++ rust/mxl/src/samples_writer.rs | 19 +++++ 5 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 rust/mxl/src/samples_write_access.rs diff --git a/rust/mxl/examples/flow-writer.rs b/rust/mxl/examples/flow-writer.rs index 336a43a27..7fe5ba138 100644 --- a/rust/mxl/examples/flow-writer.rs +++ b/rust/mxl/examples/flow-writer.rs @@ -1,9 +1,10 @@ mod common; use clap::Parser; -use mxl::config::get_mxf_so_path; use tracing::info; +use mxl::config::get_mxf_so_path; + #[derive(Debug, Parser)] #[command(version = clap::crate_version!(), author = clap::crate_authors!())] pub struct Opts { @@ -15,9 +16,14 @@ pub struct Opts { #[arg(long)] pub flow_config_file: String, - /// The number of grains to write. + /// The number of grains to write. If not specified, will run until stopped. + #[arg(long)] + pub grain_or_sample_count: Option, + + /// The number of samples to be written in one open samples call. Is only valid for "continuous" + /// flows. If not specified, will more or less fit 10 ms. #[arg(long)] - pub grain_count: Option, + pub sample_batch_size: Option, } fn main() -> Result<(), mxl::Error> { @@ -33,6 +39,29 @@ fn main() -> Result<(), mxl::Error> { )) })?; let flow_info = mxl_instance.create_flow(flow_def.as_str(), None)?; + + if flow_info.is_discrete_flow() { + if opts.sample_batch_size.is_some() { + return Err(mxl::Error::Other( + "Sample batch size is only relevant for \"continuous\" flows.".to_owned(), + )); + } + write_grains(mxl_instance, flow_info, opts.grain_or_sample_count) + } else { + write_samples( + mxl_instance, + flow_info, + opts.grain_or_sample_count, + opts.sample_batch_size, + ) + } +} + +pub fn write_grains( + mxl_instance: mxl::MxlInstance, + flow_info: mxl::FlowInfo, + grain_count: Option, +) -> Result<(), mxl::Error> { let flow_id = flow_info.common_flow_info().id().to_string(); let grain_rate = flow_info.discrete_flow_info()?.grainRate; let mut grain_index = mxl_instance.get_current_index(&grain_rate); @@ -44,7 +73,7 @@ fn main() -> Result<(), mxl::Error> { .create_flow_writer(flow_id.as_str())? .to_grain_writer()?; - let mut remaining_grains = opts.grain_count; + let mut remaining_grains = grain_count; loop { if let Some(count) = remaining_grains { if count == 0 { @@ -76,3 +105,67 @@ fn main() -> Result<(), mxl::Error> { mxl_instance.destroy_flow(flow_id.as_str())?; Ok(()) } + +pub fn write_samples( + mxl_instance: mxl::MxlInstance, + flow_info: mxl::FlowInfo, + sample_count: Option, + batch_size: Option, +) -> Result<(), mxl::Error> { + let flow_id = flow_info.common_flow_info().id().to_string(); + let sample_rate = flow_info.continuous_flow_info()?.sampleRate; + let batch_size = + batch_size.unwrap_or((sample_rate.numerator / (100 * sample_rate.denominator)) as u64); + let mut samples_index = mxl_instance.get_current_index(&sample_rate); + info!( + "Will write to flow \"{flow_id}\" with sample rate {}/{}, using batches of size {batch_size} samples, first batch ending at index {samples_index}.", + sample_rate.numerator, sample_rate.denominator + ); + let writer = mxl_instance + .create_flow_writer(flow_id.as_str())? + .to_samples_writer()?; + + let mut remaining_samples = sample_count; + loop { + if let Some(count) = remaining_samples { + if count == 0 { + break; + } + } + let samples_to_write = u64::min(batch_size, remaining_samples.unwrap_or(u64::MAX)); + if let Some(count) = remaining_samples { + remaining_samples = Some(count.saturating_sub(batch_size)); + } + + let mut samples_write_access = writer.open_samples(samples_index, batch_size as usize)?; + let mut writing_sample_index = samples_index - batch_size + 1; + for channel in 0..samples_write_access.channels() { + let (data_1, data_2) = samples_write_access.channel_data_mut(channel)?; + for i in 0..data_1.len() { + data_1[i] = (writing_sample_index % 256) as u8; + writing_sample_index += 1; + } + for i in 0..data_2.len() { + data_2[i] = (writing_sample_index % 256) as u8; + writing_sample_index += 1; + } + } + samples_write_access.commit()?; + + let timestamp = + mxl_instance.index_to_timestamp(samples_index + batch_size, &sample_rate)?; + let sleep_duration = + mxl_instance.get_duration_until_index(samples_index + batch_size, &sample_rate)?; + info!( + "Finished writing {samples_to_write} samples into batch ending with index {samples_index}, will sleep for {:?} until timestamp {timestamp}.", + sleep_duration + ); + samples_index += batch_size; + mxl_instance.sleep_for(sleep_duration); + } + + info!("Finished writing requested number of samples, deleting the flow."); + writer.destroy()?; + mxl_instance.destroy_flow(flow_id.as_str())?; + Ok(()) +} diff --git a/rust/mxl/src/flow.rs b/rust/mxl/src/flow.rs index b0c63933e..cfda4d845 100644 --- a/rust/mxl/src/flow.rs +++ b/rust/mxl/src/flow.rs @@ -44,9 +44,23 @@ impl FlowInfo { Ok(unsafe { &self.value.__bindgen_anon_1.discrete }) } + pub fn continuous_flow_info(&self) -> Result<&mxl_sys::ContinuousFlowInfo> { + if is_discrete_data_format(self.value.common.format) { + return Err(Error::Other(format!( + "Flow format is {}, audio required.", + self.value.common.format + ))); + } + Ok(unsafe { &self.value.__bindgen_anon_1.continuous }) + } + pub fn common_flow_info(&self) -> CommonFlowInfo { CommonFlowInfo(&self.value.common) } + + pub fn is_discrete_flow(&self) -> bool { + is_discrete_data_format(self.value.common.format) + } } pub struct CommonFlowInfo<'a>(&'a mxl_sys::CommonFlowInfo); diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index 5167a50d6..d55eb0dba 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -7,6 +7,7 @@ mod grain_data; mod grain_write_access; mod grain_writer; mod instance; +mod samples_write_access; mod samples_writer; pub mod config; @@ -20,3 +21,4 @@ pub use flow_writer::MxlFlowWriter; pub use grain_data::*; pub use grain_write_access::GrainWriteAccess; pub use instance::MxlInstance; +pub use samples_write_access::SamplesWriteAccess; diff --git a/rust/mxl/src/samples_write_access.rs b/rust/mxl/src/samples_write_access.rs new file mode 100644 index 000000000..e9680950b --- /dev/null +++ b/rust/mxl/src/samples_write_access.rs @@ -0,0 +1,95 @@ +use std::marker::PhantomData; +use std::sync::Arc; + +use tracing::error; + +use crate::Error; +use crate::instance::InstanceContext; + +/// RAII samples writing session +/// +/// Automatically cancels the samples if not explicitly committed. +/// +/// The data may be split into 2 different buffer slices in case of a wrapped ring. Provides access +/// either directly to the slices or to individual samples by index inside the batch. +pub struct SamplesWriteAccess<'a> { + context: Arc, + writer: mxl_sys::mxlFlowWriter, + buffer_slice: mxl_sys::MutableWrappedMultiBufferSlice, + /// Serves as a flag to know whether to cancel the samples on drop. + committed_or_canceled: bool, + phantom: PhantomData<&'a ()>, +} + +impl<'a> SamplesWriteAccess<'a> { + pub(crate) fn new( + context: Arc, + writer: mxl_sys::mxlFlowWriter, + buffer_slice: mxl_sys::MutableWrappedMultiBufferSlice, + ) -> Self { + Self { + context, + writer, + buffer_slice, + committed_or_canceled: false, + phantom: PhantomData, + } + } + + pub fn commit(mut self) -> crate::Result<()> { + self.committed_or_canceled = true; + + unsafe { Error::from_status(self.context.api.mxl_flow_writer_commit_samples(self.writer)) } + } + + /// Please note that the behavior of canceling samples writing is dependent on the behavior + /// implemented in MXL itself. Particularly, if samples data have been mutated and then writing + /// canceled, mutation will most likely stay in place, only head won't be updated, and readers + /// notified. + pub fn cancel(mut self) -> crate::Result<()> { + self.committed_or_canceled = true; + + unsafe { Error::from_status(self.context.api.mxl_flow_writer_cancel_samples(self.writer)) } + } + + pub fn channels(&self) -> usize { + self.buffer_slice.count + } + + /// Provides direct access to buffer of the given channel. The access is split into two slices + /// to cover cases when the ring is not continuous. + /// + /// Currently, we provide just raw bytes access. Probably we should provide some sample-based + /// access and some index-based access (where we hide the complexity of 2 slices) as well? + /// + /// Samples are f32? + pub fn channel_data_mut(&mut self, channel: usize) -> crate::Result<(&mut [u8], &mut [u8])> { + if channel >= self.buffer_slice.count { + return Err(Error::InvalidArg); + } + unsafe { + let ptr_1 = (self.buffer_slice.base.fragments[0].pointer as *mut u8) + .add(self.buffer_slice.stride * channel); + let size_1 = self.buffer_slice.base.fragments[0].size; + let ptr_2 = (self.buffer_slice.base.fragments[1].pointer as *mut u8) + .add(self.buffer_slice.stride * channel); + let size_2 = self.buffer_slice.base.fragments[1].size; + Ok(( + std::slice::from_raw_parts_mut(ptr_1, size_1), + std::slice::from_raw_parts_mut(ptr_2, size_2), + )) + } + } +} + +impl<'a> Drop for SamplesWriteAccess<'a> { + fn drop(&mut self) { + if !self.committed_or_canceled { + if let Err(error) = unsafe { + Error::from_status(self.context.api.mxl_flow_writer_cancel_samples(self.writer)) + } { + error!("Failed to cancel grain write on drop: {:?}", error); + } + } + } +} diff --git a/rust/mxl/src/samples_writer.rs b/rust/mxl/src/samples_writer.rs index e14b88a59..40cbe41c8 100644 --- a/rust/mxl/src/samples_writer.rs +++ b/rust/mxl/src/samples_writer.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use crate::samples_write_access::SamplesWriteAccess; use crate::{Error, Result, instance::InstanceContext}; /// MXL Flow Writer for continuous flows (samples-based data like audio) @@ -17,6 +18,24 @@ impl SamplesWriter { self.destroy_inner() } + pub fn open_samples<'a>(&'a self, index: u64, count: usize) -> Result> { + let mut buffer_slice: mxl_sys::MutableWrappedMultiBufferSlice = + unsafe { std::mem::zeroed() }; + unsafe { + Error::from_status(self.context.api.mxl_flow_writer_open_samples( + self.writer, + index, + count, + &mut buffer_slice, + ))?; + } + Ok(SamplesWriteAccess::new( + self.context.clone(), + self.writer, + buffer_slice, + )) + } + fn destroy_inner(&mut self) -> Result<()> { if self.writer.is_null() { return Err(Error::InvalidArg); From 5b8697d58b6ac64285e4814c18ba8e57ff896029 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Thu, 17 Jul 2025 14:18:47 +0200 Subject: [PATCH 27/77] Change MxlFlowReader into a factory class for more specific readers - Added grain reader and a skeleton for samples reader. Signed-off-by: Pavel Cernohorsky --- rust/mxl/examples/flow-reader.rs | 20 ++++++ rust/mxl/src/flow_reader.rs | 110 ++++++++++++------------------- rust/mxl/src/grain_reader.rs | 91 +++++++++++++++++++++++++ rust/mxl/src/lib.rs | 4 ++ rust/mxl/src/samples_reader.rs | 48 ++++++++++++++ rust/mxl/tests/basic_tests.rs | 7 +- 6 files changed, 209 insertions(+), 71 deletions(-) create mode 100644 rust/mxl/src/grain_reader.rs create mode 100644 rust/mxl/src/samples_reader.rs diff --git a/rust/mxl/examples/flow-reader.rs b/rust/mxl/examples/flow-reader.rs index 1b222b0d3..b6920337e 100644 --- a/rust/mxl/examples/flow-reader.rs +++ b/rust/mxl/examples/flow-reader.rs @@ -28,6 +28,18 @@ fn main() -> Result<(), mxl::Error> { let mxl_instance = mxl::MxlInstance::new(mxl_api, &opts.mxl_domain, "")?; let reader = mxl_instance.create_flow_reader(&opts.flow_id)?; let flow_info = reader.get_info()?; + if flow_info.is_discrete_flow() { + read_grains(mxl_instance, reader.to_grain_reader()?, flow_info) + } else { + read_samples(mxl_instance, reader.to_samples_reader()?, flow_info) + } +} + +fn read_grains( + mxl_instance: mxl::MxlInstance, + reader: mxl::GrainReader, + flow_info: mxl::FlowInfo, +) -> Result<(), mxl::Error> { let rate = flow_info.discrete_flow_info()?.grainRate; let current_index = mxl_instance.get_current_index(&rate); @@ -43,3 +55,11 @@ fn main() -> Result<(), mxl::Error> { Ok(()) } + +fn read_samples( + _mxl_instance: mxl::MxlInstance, + _reader: mxl::SamplesReader, + _flow_info: mxl::FlowInfo, +) -> Result<(), mxl::Error> { + todo!() +} diff --git a/rust/mxl/src/flow_reader.rs b/rust/mxl/src/flow_reader.rs index 66b16b362..26991b331 100644 --- a/rust/mxl/src/flow_reader.rs +++ b/rust/mxl/src/flow_reader.rs @@ -1,96 +1,70 @@ -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; -use crate::{Error, GrainData, Result, flow::FlowInfo, instance::InstanceContext}; +use crate::flow::is_discrete_data_format; +use crate::grain_reader::GrainReader; +use crate::samples_reader::SamplesReader; +use crate::{DataFormat, Error, Result, flow::FlowInfo, instance::InstanceContext}; pub struct MxlFlowReader { context: Arc, reader: mxl_sys::mxlFlowReader, } +pub(crate) fn get_flow_info( + context: &Arc, + reader: mxl_sys::mxlFlowReader, +) -> Result { + let mut flow_info: mxl_sys::FlowInfo = unsafe { std::mem::zeroed() }; + unsafe { + Error::from_status(context.api.mxl_flow_reader_get_info(reader, &mut flow_info))?; + } + Ok(FlowInfo { value: flow_info }) +} + impl MxlFlowReader { pub(crate) fn new(context: Arc, reader: mxl_sys::mxlFlowReader) -> Self { Self { context, reader } } - pub fn destroy(mut self) -> Result<()> { - self.destroy_inner() - } - pub fn get_info(&self) -> Result { - let mut flow_info: mxl_sys::FlowInfo = unsafe { std::mem::zeroed() }; - unsafe { - Error::from_status( - self.context - .api - .mxl_flow_reader_get_info(self.reader, &mut flow_info), - )?; - } - Ok(FlowInfo { value: flow_info }) + get_flow_info(&self.context, self.reader) } - pub fn get_complete_grain<'a>( - &'a self, - index: u64, - timeout: Duration, - ) -> Result> { - let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; - let mut payload_ptr: *mut u8 = std::ptr::null_mut(); - let timeout_ns = timeout.as_nanos() as u64; - loop { - unsafe { - Error::from_status(self.context.api.mxl_flow_reader_get_grain( - self.reader, - index, - timeout_ns, - &mut grain_info, - &mut payload_ptr, - ))?; - } - if grain_info.commitedSize != grain_info.grainSize { - // We don't need partial grains. Wait for the grain to be complete. - continue; - } - if payload_ptr.is_null() { - return Err(Error::Other(format!( - "Failed to get grain payload for index {}.", - index - ))); - } - break; + pub fn to_grain_reader(mut self) -> Result { + let flow_type = self.get_info()?.value.common.format; + if !is_discrete_data_format(flow_type) { + return Err(Error::Other(format!( + "Cannot convert MxlFlowReader to GrainReader for continuous flow of type \"{:?}\".", + DataFormat::from(flow_type) + ))); } - - // SAFETY - // We know that the lifetime is as long as the flow, so it is at least self's lifetime. - // It may happen that the buffer is overwritten by a subsequent write, but it is safe. - let user_data: &'a [u8] = - unsafe { std::mem::transmute::<&[u8], &'a [u8]>(&grain_info.userData) }; - - let payload = - unsafe { std::slice::from_raw_parts(payload_ptr, grain_info.grainSize as usize) }; - - Ok(GrainData { user_data, payload }) + let result = GrainReader::new(self.context.clone(), self.reader); + self.reader = std::ptr::null_mut(); + Ok(result) } - fn destroy_inner(&mut self) -> Result<()> { - if self.reader.is_null() { - return Err(Error::InvalidArg); + pub fn to_samples_reader(mut self) -> Result { + let flow_type = self.get_info()?.value.common.format; + if is_discrete_data_format(flow_type) { + return Err(Error::Other(format!( + "Cannot convert MxlFlowReader to SamplesReader for discrete flow of type \"{:?}\".", + DataFormat::from(flow_type) + ))); } - - let mut reader = std::ptr::null_mut(); - std::mem::swap(&mut self.reader, &mut reader); - - Error::from_status(unsafe { - self.context - .api - .mxl_release_flow_reader(self.context.instance, reader) - }) + let result = SamplesReader::new(self.context.clone(), self.reader); + self.reader = std::ptr::null_mut(); + Ok(result) } } impl Drop for MxlFlowReader { fn drop(&mut self) { if !self.reader.is_null() { - if let Err(err) = self.destroy_inner() { + if let Err(err) = Error::from_status(unsafe { + self.context + .api + .mxl_release_flow_reader(self.context.instance, self.reader) + }) { tracing::error!("Failed to release MXL flow reader: {:?}", err); } } diff --git a/rust/mxl/src/grain_reader.rs b/rust/mxl/src/grain_reader.rs new file mode 100644 index 000000000..a1504ecef --- /dev/null +++ b/rust/mxl/src/grain_reader.rs @@ -0,0 +1,91 @@ +use std::{sync::Arc, time::Duration}; + +use crate::flow_reader::get_flow_info; +use crate::{Error, GrainData, Result, flow::FlowInfo, instance::InstanceContext}; + +pub struct GrainReader { + context: Arc, + reader: mxl_sys::mxlFlowReader, +} + +impl GrainReader { + pub(crate) fn new(context: Arc, reader: mxl_sys::mxlFlowReader) -> Self { + Self { context, reader } + } + + pub fn destroy(mut self) -> Result<()> { + self.destroy_inner() + } + + pub fn get_info(&self) -> Result { + get_flow_info(&self.context, self.reader) + } + + pub fn get_complete_grain<'a>( + &'a self, + index: u64, + timeout: Duration, + ) -> Result> { + let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; + let mut payload_ptr: *mut u8 = std::ptr::null_mut(); + let timeout_ns = timeout.as_nanos() as u64; + loop { + unsafe { + Error::from_status(self.context.api.mxl_flow_reader_get_grain( + self.reader, + index, + timeout_ns, + &mut grain_info, + &mut payload_ptr, + ))?; + } + if grain_info.commitedSize != grain_info.grainSize { + // We don't need partial grains. Wait for the grain to be complete. + continue; + } + if payload_ptr.is_null() { + return Err(Error::Other(format!( + "Failed to get grain payload for index {}.", + index + ))); + } + break; + } + + // SAFETY + // We know that the lifetime is as long as the flow, so it is at least self's lifetime. + // It may happen that the buffer is overwritten by a subsequent write, but it is safe. + let user_data: &'a [u8] = + unsafe { std::mem::transmute::<&[u8], &'a [u8]>(&grain_info.userData) }; + + let payload = + unsafe { std::slice::from_raw_parts(payload_ptr, grain_info.grainSize as usize) }; + + Ok(GrainData { user_data, payload }) + } + + fn destroy_inner(&mut self) -> Result<()> { + if self.reader.is_null() { + return Err(Error::InvalidArg); + } + + let mut reader = std::ptr::null_mut(); + std::mem::swap(&mut self.reader, &mut reader); + + Error::from_status(unsafe { + self.context + .api + .mxl_release_flow_reader(self.context.instance, reader) + }) + } +} + +impl Drop for GrainReader { + fn drop(&mut self) { + if !self.reader.is_null() { + if let Err(err) = self.destroy_inner() { + tracing::error!("Failed to release MXL flow reader (discrete): {:?}", err); + } + } + } +} diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index d55eb0dba..045f2abe8 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -4,9 +4,11 @@ mod flow; mod flow_reader; mod flow_writer; mod grain_data; +mod grain_reader; mod grain_write_access; mod grain_writer; mod instance; +mod samples_reader; mod samples_write_access; mod samples_writer; @@ -19,6 +21,8 @@ pub use flow::*; pub use flow_reader::MxlFlowReader; pub use flow_writer::MxlFlowWriter; pub use grain_data::*; +pub use grain_reader::GrainReader; pub use grain_write_access::GrainWriteAccess; pub use instance::MxlInstance; +pub use samples_reader::SamplesReader; pub use samples_write_access::SamplesWriteAccess; diff --git a/rust/mxl/src/samples_reader.rs b/rust/mxl/src/samples_reader.rs new file mode 100644 index 000000000..20cb1aea1 --- /dev/null +++ b/rust/mxl/src/samples_reader.rs @@ -0,0 +1,48 @@ +use std::sync::Arc; + +use crate::flow_reader::get_flow_info; +use crate::{Error, Result, flow::FlowInfo, instance::InstanceContext}; + +pub struct SamplesReader { + context: Arc, + reader: mxl_sys::mxlFlowReader, +} + +impl SamplesReader { + pub(crate) fn new(context: Arc, reader: mxl_sys::mxlFlowReader) -> Self { + Self { context, reader } + } + + pub fn destroy(mut self) -> Result<()> { + self.destroy_inner() + } + + pub fn get_info(&self) -> Result { + get_flow_info(&self.context, self.reader) + } + + fn destroy_inner(&mut self) -> Result<()> { + if self.reader.is_null() { + return Err(Error::InvalidArg); + } + + let mut reader = std::ptr::null_mut(); + std::mem::swap(&mut self.reader, &mut reader); + + Error::from_status(unsafe { + self.context + .api + .mxl_release_flow_reader(self.context.instance, reader) + }) + } +} + +impl Drop for SamplesReader { + fn drop(&mut self) { + if !self.reader.is_null() { + if let Err(err) = self.destroy_inner() { + tracing::error!("Failed to release MXL flow reader (continuous): {:?}", err); + } + } + } +} diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index 762282e20..0eebbe21c 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -40,7 +40,7 @@ fn setup_test() -> mxl::MxlInstance { } #[test] -fn basic_mxl_writing_reading() { +fn basic_mxl_grain_writing_reading() { let mxl_instance = setup_test(); let flow_config_file = mxl::config::get_mxl_repo_root().join("lib/tests/data/v210_flow.json"); let flow_def = mxl::tools::read_file(flow_config_file.as_path()) @@ -57,17 +57,18 @@ fn basic_mxl_writing_reading() { let flow_writer = mxl_instance.create_flow_writer(flow_id.as_str()).unwrap(); let grain_writer = flow_writer.to_grain_writer().unwrap(); let flow_reader = mxl_instance.create_flow_reader(flow_id.as_str()).unwrap(); + let grain_reader = flow_reader.to_grain_reader().unwrap(); let rate = flow_info.discrete_flow_info().unwrap().grainRate; let current_index = mxl_instance.get_current_index(&rate); let grain_write_access = grain_writer.open_grain(current_index).unwrap(); let grain_size = grain_write_access.max_size(); grain_write_access.commit(grain_size).unwrap(); - let grain_data = flow_reader + let grain_data = grain_reader .get_complete_grain(current_index, Duration::from_secs(5)) .unwrap(); let grain_data: OwnedGrainData = grain_data.into(); info!("Grain data len: {:?}", grain_data.payload.len()); - flow_reader.destroy().unwrap(); + grain_reader.destroy().unwrap(); grain_writer.destroy().unwrap(); mxl_instance.destroy_flow(flow_id.as_str()).unwrap(); unsafe { mxl_instance.destroy() }.unwrap(); From 1efbf67d9512467ab6ed16993a1f21c10bf667ef Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Fri, 18 Jul 2025 12:36:33 +0200 Subject: [PATCH 28/77] Add basic continuous flow reader Rust wrapper Signed-off-by: Pavel Cernohorsky --- rust/mxl/examples/flow-reader.rs | 103 +++++++++++++++++++++++++++++-- rust/mxl/src/lib.rs | 2 + rust/mxl/src/samples_data.rs | 74 ++++++++++++++++++++++ rust/mxl/src/samples_reader.rs | 14 +++++ rust/mxl/tests/basic_tests.rs | 65 ++++++++++++++----- 5 files changed, 235 insertions(+), 23 deletions(-) create mode 100644 rust/mxl/src/samples_data.rs diff --git a/rust/mxl/examples/flow-reader.rs b/rust/mxl/examples/flow-reader.rs index b6920337e..98bd2a2c0 100644 --- a/rust/mxl/examples/flow-reader.rs +++ b/rust/mxl/examples/flow-reader.rs @@ -4,7 +4,7 @@ use std::time::Duration; use clap::Parser; use mxl::config::get_mxf_so_path; -use tracing::info; +use tracing::{info, warn}; const READ_TIMEOUT: Duration = Duration::from_secs(5); @@ -18,6 +18,12 @@ pub struct Opts { /// The id of the flow to read. #[arg(long)] pub flow_id: String, + + /// The number of samples to be read in one open samples call. Is only valid for "continuous" + /// flows. If not specified, the value provided by the writer will be used if available, or this + /// will more or less fit 10 ms as a fallback. + #[arg(long)] + pub sample_batch_size: Option, } fn main() -> Result<(), mxl::Error> { @@ -29,9 +35,19 @@ fn main() -> Result<(), mxl::Error> { let reader = mxl_instance.create_flow_reader(&opts.flow_id)?; let flow_info = reader.get_info()?; if flow_info.is_discrete_flow() { + if opts.sample_batch_size.is_some() { + return Err(mxl::Error::Other( + "Sample batch size is only relevant for \"continuous\" flows.".to_owned(), + )); + } read_grains(mxl_instance, reader.to_grain_reader()?, flow_info) } else { - read_samples(mxl_instance, reader.to_samples_reader()?, flow_info) + read_samples( + mxl_instance, + reader.to_samples_reader()?, + flow_info, + opts.sample_batch_size, + ) } } @@ -57,9 +73,84 @@ fn read_grains( } fn read_samples( - _mxl_instance: mxl::MxlInstance, - _reader: mxl::SamplesReader, - _flow_info: mxl::FlowInfo, + mxl_instance: mxl::MxlInstance, + reader: mxl::SamplesReader, + flow_info: mxl::FlowInfo, + batch_size: Option, ) -> Result<(), mxl::Error> { - todo!() + let flow_id = flow_info.common_flow_info().id().to_string(); + let sample_rate = flow_info.continuous_flow_info()?.sampleRate; + let continous_flow_info = flow_info.continuous_flow_info()?; + let batch_size = if let Some(batch_size) = batch_size { + if continous_flow_info.commitBatchSize != 0 + && batch_size != continous_flow_info.commitBatchSize as u64 + { + warn!( + "Writer batch size is set to {}, but sample batch size is provided, using the \ + latter.", + continous_flow_info.commitBatchSize + ); + } + batch_size as usize + } else { + if continous_flow_info.commitBatchSize == 0 { + let batch_size = (sample_rate.numerator / (100 * sample_rate.denominator)) as usize; + warn!( + "Writer batch size not available, using fallback value of {}.", + batch_size + ); + batch_size + } else { + continous_flow_info.commitBatchSize as usize + } + }; + let mut read_head = reader.get_info()?.continuous_flow_info()?.headIndex; + let mut read_head_valid_at = mxl_instance.get_time(); + info!( + "Will read from flow \"{flow_id}\" with sample rate {}/{}, using batches of size \ + {batch_size} samples, first batch ending at index {read_head}.", + sample_rate.numerator, sample_rate.denominator + ); + loop { + let samples_data = reader.get_samples(read_head, batch_size)?; + info!( + "Read samples for {} channel(s) at index {}.", + samples_data.num_of_channels(), + read_head + ); + if samples_data.num_of_channels() > 0 { + let channel_data = samples_data.channel_data(0)?; + info!( + "Buffer size for channel 0 is ({}, {}).", + channel_data.0.len(), + channel_data.1.len() + ); + } + // MXL currently does not have any samples reading mechanism which would wait for data to be + // available. + // We will just blindly assume that more data will be available when we need them. + // This, of course, does not have to be true, because the write batch size may be larger + // than our reading batch size. + let next_head = read_head + batch_size as u64; + let next_head_timestamp = mxl_instance.index_to_timestamp(next_head, &sample_rate)?; + let read_head_timestamp = mxl_instance.index_to_timestamp(read_head, &sample_rate)?; + let read_batch_duration = next_head_timestamp - read_head_timestamp; + let deadline = std::time::Instant::now() + READ_TIMEOUT; + loop { + read_head_valid_at += read_batch_duration; + let sleep_duration = + Duration::from_nanos(read_head_valid_at.saturating_sub(mxl_instance.get_time())); + info!("Will sleep for {:?}.", sleep_duration); + mxl_instance.sleep_for(sleep_duration); + if std::time::Instant::now() >= deadline { + warn!("Timeout while waiting for samples at index {}.", next_head); + return Err(mxl::Error::Timeout); + } + let available_head = reader.get_info()?.continuous_flow_info()?.headIndex; + if available_head >= next_head { + break; + } + } + read_head = next_head; + } } diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index 045f2abe8..2cb45aad3 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -8,6 +8,7 @@ mod grain_reader; mod grain_write_access; mod grain_writer; mod instance; +mod samples_data; mod samples_reader; mod samples_write_access; mod samples_writer; @@ -24,5 +25,6 @@ pub use grain_data::*; pub use grain_reader::GrainReader; pub use grain_write_access::GrainWriteAccess; pub use instance::MxlInstance; +pub use samples_data::*; pub use samples_reader::SamplesReader; pub use samples_write_access::SamplesWriteAccess; diff --git a/rust/mxl/src/samples_data.rs b/rust/mxl/src/samples_data.rs new file mode 100644 index 000000000..f1d005b6a --- /dev/null +++ b/rust/mxl/src/samples_data.rs @@ -0,0 +1,74 @@ +use crate::Error; +use std::marker::PhantomData; + +pub struct SamplesData<'a> { + buffer_slice: mxl_sys::WrappedMultiBufferSlice, + phantom: PhantomData<&'a ()>, +} + +impl<'a> SamplesData<'a> { + pub(crate) fn new(buffer_slice: mxl_sys::WrappedMultiBufferSlice) -> Self { + Self { + buffer_slice, + phantom: Default::default(), + } + } + + pub fn num_of_channels(&self) -> usize { + self.buffer_slice.count + } + + pub fn channel_data(&self, channel: usize) -> crate::Result<(&[u8], &[u8])> { + if channel >= self.buffer_slice.count { + return Err(Error::InvalidArg); + } + unsafe { + let ptr_1 = (self.buffer_slice.base.fragments[0].pointer as *const u8) + .add(self.buffer_slice.stride * channel); + let size_1 = self.buffer_slice.base.fragments[0].size; + let ptr_2 = (self.buffer_slice.base.fragments[1].pointer as *const u8) + .add(self.buffer_slice.stride * channel); + let size_2 = self.buffer_slice.base.fragments[1].size; + Ok(( + std::slice::from_raw_parts(ptr_1, size_1), + std::slice::from_raw_parts(ptr_2, size_2), + )) + } + } + + pub fn to_owned(&self) -> OwnedSamplesData { + self.into() + } +} + +impl<'a> AsRef> for SamplesData<'a> { + fn as_ref(&self) -> &SamplesData<'a> { + self + } +} + +pub struct OwnedSamplesData { + /// Data belonging to each of the channels. + pub payload: Vec>, +} + +impl<'a> From<&SamplesData<'a>> for OwnedSamplesData { + fn from(value: &SamplesData<'a>) -> Self { + let mut payload = Vec::with_capacity(value.buffer_slice.count); + for channel in 0..value.buffer_slice.count { + // The following unwrap is safe because the channel index always stays in the valid range. + let (data_1, data_2) = value.channel_data(channel).unwrap(); + let mut channel_payload = Vec::with_capacity(data_1.len() + data_2.len()); + channel_payload.extend(data_1); + channel_payload.extend(data_2); + payload.push(channel_payload); + } + Self { payload } + } +} + +impl<'a> From> for OwnedSamplesData { + fn from(value: SamplesData<'a>) -> Self { + value.as_ref().into() + } +} diff --git a/rust/mxl/src/samples_reader.rs b/rust/mxl/src/samples_reader.rs index 20cb1aea1..e872afa60 100644 --- a/rust/mxl/src/samples_reader.rs +++ b/rust/mxl/src/samples_reader.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use crate::flow_reader::get_flow_info; +use crate::samples_data::SamplesData; use crate::{Error, Result, flow::FlowInfo, instance::InstanceContext}; pub struct SamplesReader { @@ -21,6 +22,19 @@ impl SamplesReader { get_flow_info(&self.context, self.reader) } + pub fn get_samples(&self, index: u64, count: usize) -> Result { + let mut buffer_slice: mxl_sys::WrappedMultiBufferSlice = unsafe { std::mem::zeroed() }; + unsafe { + Error::from_status(self.context.api.mxl_flow_reader_get_samples( + self.reader, + index, + count, + &mut buffer_slice, + ))?; + } + Ok(SamplesData::new(buffer_slice)) + } + fn destroy_inner(&mut self) -> Result<()> { if self.reader.is_null() { return Err(Error::InvalidArg); diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index 0eebbe21c..3292dde9d 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -4,24 +4,22 @@ /// change in the future. For now, feel free to just edit the path to your library. use std::time::Duration; -use mxl::{OwnedGrainData, config::get_mxf_so_path}; +use mxl::{MxlInstance, OwnedGrainData, OwnedSamplesData, config::get_mxf_so_path}; use tracing::info; static LOG_ONCE: std::sync::Once = std::sync::Once::new(); -fn setup_empty_domain() -> String { - // TODO: Randomize the domain name to allow parallel tests run? - // On the other hand, some tests may leave stuff behind, this way we do not keep - // increasing garbage from the tests. - let result = "/dev/shm/mxl_rust_unit_tests_domain"; - if std::path::Path::new(result).exists() { - std::fs::remove_dir_all(result).expect("Failed to remove existing test domain directory"); +fn setup_empty_domain(test: &str) -> String { + let result = format!("/dev/shm/mxl_rust_unit_tests_domain_{}", test); + if std::path::Path::new(result.as_str()).exists() { + std::fs::remove_dir_all(result.as_str()) + .expect("Failed to remove existing test domain directory"); } - std::fs::create_dir_all(result).expect("Failed to create test domain directory"); - result.to_string() + std::fs::create_dir_all(result.as_str()).expect("Failed to create test domain directory"); + result } -fn setup_test() -> mxl::MxlInstance { +fn setup_test(test: &str) -> mxl::MxlInstance { // Set up the logging to use the RUST_LOG environment variable and if not present, print INFO // and higher. LOG_ONCE.call_once(|| { @@ -35,14 +33,15 @@ fn setup_test() -> mxl::MxlInstance { }); let mxl_api = mxl::load_api(get_mxf_so_path()).unwrap(); - let domain = setup_empty_domain(); + let domain = setup_empty_domain(test); mxl::MxlInstance::new(mxl_api, &domain, "").unwrap() } -#[test] -fn basic_mxl_grain_writing_reading() { - let mxl_instance = setup_test(); - let flow_config_file = mxl::config::get_mxl_repo_root().join("lib/tests/data/v210_flow.json"); +fn prepare_flow_info>( + mxl_instance: &MxlInstance, + path: P, +) -> mxl::FlowInfo { + let flow_config_file = mxl::config::get_mxl_repo_root().join(path); let flow_def = mxl::tools::read_file(flow_config_file.as_path()) .map_err(|error| { mxl::Error::Other(format!( @@ -52,7 +51,13 @@ fn basic_mxl_grain_writing_reading() { )) }) .unwrap(); - let flow_info = mxl_instance.create_flow(flow_def.as_str(), None).unwrap(); + mxl_instance.create_flow(flow_def.as_str(), None).unwrap() +} + +#[test] +fn basic_mxl_grain_writing_reading() { + let mxl_instance = setup_test("grains"); + let flow_info = prepare_flow_info(&mxl_instance, "lib/tests/data/v210_flow.json"); let flow_id = flow_info.common_flow_info().id().to_string(); let flow_writer = mxl_instance.create_flow_writer(flow_id.as_str()).unwrap(); let grain_writer = flow_writer.to_grain_writer().unwrap(); @@ -73,3 +78,29 @@ fn basic_mxl_grain_writing_reading() { mxl_instance.destroy_flow(flow_id.as_str()).unwrap(); unsafe { mxl_instance.destroy() }.unwrap(); } + +#[test] +fn basic_mxl_samples_writing_reading() { + let mxl_instance = setup_test("samples"); + let flow_info = prepare_flow_info(&mxl_instance, "lib/tests/data/audio_flow.json"); + let flow_id = flow_info.common_flow_info().id().to_string(); + let flow_writer = mxl_instance.create_flow_writer(flow_id.as_str()).unwrap(); + let samples_writer = flow_writer.to_samples_writer().unwrap(); + let flow_reader = mxl_instance.create_flow_reader(flow_id.as_str()).unwrap(); + let samples_reader = flow_reader.to_samples_reader().unwrap(); + let rate = flow_info.continuous_flow_info().unwrap().sampleRate; + let current_index = mxl_instance.get_current_index(&rate); + let samples_write_access = samples_writer.open_samples(current_index, 42).unwrap(); + samples_write_access.commit().unwrap(); + let samples_data = samples_reader.get_samples(current_index, 42).unwrap(); + let samples_data: OwnedSamplesData = samples_data.into(); + info!( + "Samples data contains {} channels(s), channel 0 has {} byte(s).", + samples_data.payload.len(), + samples_data.payload[0].len() + ); + samples_reader.destroy().unwrap(); + samples_writer.destroy().unwrap(); + mxl_instance.destroy_flow(flow_id.as_str()).unwrap(); + unsafe { mxl_instance.destroy() }.unwrap(); +} From 66e2000d63ef92f9b1d5f172a7584c22db5557b2 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Fri, 18 Jul 2025 12:43:53 +0200 Subject: [PATCH 29/77] Convert instance destroy into safe function Signed-off-by: Pavel Cernohorsky --- rust/mxl/src/instance.rs | 29 +++++++++-------------------- rust/mxl/tests/basic_tests.rs | 4 ++-- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index eab1c6ac4..ca712c2f2 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -19,14 +19,8 @@ unsafe impl Sync for InstanceContext {} impl InstanceContext { /// This function forces the destruction of the MXL instance. - /// It is not safe as other objects may still be using it. - /// It is meant for testing purposes only. - /// - /// # Safety - /// - /// The caller must ensure that no other objects are using the MXL instance when this function is called. - /// Calling this function while other references exist may lead to undefined behavior. - pub unsafe fn destroy(&mut self) -> Result<()> { + /// It is meant mainly for testing purposes. + pub fn destroy(mut self) -> Result<()> { unsafe { let mut instance = std::ptr::null_mut(); std::mem::swap(&mut self.instance, &mut instance); @@ -201,17 +195,12 @@ impl MxlInstance { } /// This function forces the destruction of the MXL instance. - /// It is not safe as other objects may still be using it. - /// It is meant for testing purposes only. - /// - /// # Safety - /// - /// The caller must ensure that no other objects are using the MXL instance when this function is called. - /// Calling this function while other references exist will lead to panic. - pub unsafe fn destroy(self) -> Result<()> { - unsafe { - let mut context = Arc::into_inner(self.context).unwrap(); - context.destroy() - } + /// It is meant mainly for testing purposes. + /// The caller must ensure that no other objects are using the MXL instance when this function + /// is called. + pub fn destroy(self) -> Result<()> { + let context = Arc::into_inner(self.context) + .ok_or_else(|| Error::Other("Instance is still in use.".to_string()))?; + context.destroy() } } diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index 3292dde9d..7242077ca 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -76,7 +76,7 @@ fn basic_mxl_grain_writing_reading() { grain_reader.destroy().unwrap(); grain_writer.destroy().unwrap(); mxl_instance.destroy_flow(flow_id.as_str()).unwrap(); - unsafe { mxl_instance.destroy() }.unwrap(); + mxl_instance.destroy().unwrap(); } #[test] @@ -102,5 +102,5 @@ fn basic_mxl_samples_writing_reading() { samples_reader.destroy().unwrap(); samples_writer.destroy().unwrap(); mxl_instance.destroy_flow(flow_id.as_str()).unwrap(); - unsafe { mxl_instance.destroy() }.unwrap(); + mxl_instance.destroy().unwrap(); } From 50244ab0fe218eee99e941a75c29f33c16dfbc0d Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Tue, 22 Jul 2025 10:37:20 +0200 Subject: [PATCH 30/77] Allow referencing rust crates without the need to build MXL - Currently MXL library generates a header file with it's version during the build process. This header file is not strictly needed for the Rust bindings at the moment, as it contains just constants representing the library version. - Adding a feature flag, which allows skipping this file and thus referencing MXL Rust bindings from other projects just directly from GitHub, without the need to locally build the MXL repo or even "worse" building the MXL repo fully together with bindings. Signed-off-by: Pavel Cernohorsky --- rust/mxl-sys/Cargo.toml | 3 + rust/mxl-sys/build.rs | 57 +++++++++++++++---- rust/mxl-sys/wrapper-with-version-h.h | 2 + ...{wrapper.h => wrapper-without-version-h.h} | 1 - rust/mxl/Cargo.toml | 3 + 5 files changed, 54 insertions(+), 12 deletions(-) create mode 100644 rust/mxl-sys/wrapper-with-version-h.h rename rust/mxl-sys/{wrapper.h => wrapper-without-version-h.h} (87%) diff --git a/rust/mxl-sys/Cargo.toml b/rust/mxl-sys/Cargo.toml index 308060e71..6bbb0ffdd 100644 --- a/rust/mxl-sys/Cargo.toml +++ b/rust/mxl-sys/Cargo.toml @@ -8,3 +8,6 @@ version.workspace = true [build-dependencies] bindgen.workspace = true + +[features] +mxl-not-built = [] diff --git a/rust/mxl-sys/build.rs b/rust/mxl-sys/build.rs index f9a7a6e3e..0d76aff09 100644 --- a/rust/mxl-sys/build.rs +++ b/rust/mxl-sys/build.rs @@ -6,24 +6,59 @@ const BUILD_VARIANT: &str = "Linux-Clang-Debug"; #[cfg(not(debug_assertions))] const BUILD_VARIANT: &str = "Linux-Clang-Release"; -fn main() { +struct BindgenSpecs { + header: String, + includes_dirs: Vec, +} + +fn get_bindgen_specs() -> BindgenSpecs { + #[cfg(not(feature = "mxl-not-built"))] + let header = "wrapper-with-version-h.h".to_string(); + #[cfg(feature = "mxl-not-built")] + let header = "wrapper-without-version-h.h".to_string(); + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("failed to get current directory")); let repo_root = manifest_dir.parent().unwrap().parent().unwrap(); - let build_dir = repo_root.join("build").join(BUILD_VARIANT); - let includes_dir = repo_root.join("lib").join("include"); + let mut includes_dirs = vec![ + repo_root + .join("lib") + .join("include") + .to_string_lossy() + .to_string(), + ]; + #[cfg(not(feature = "mxl-not-built"))] + { + let build_dir = repo_root.join("build").join(BUILD_VARIANT); + let build_version_dir = build_dir + .join("lib") + .join("include") + .to_string_lossy() + .to_string(); - let build_version_dir = build_dir.join("lib").join("include"); - let build_version_dir = build_version_dir.to_string_lossy(); - println!("cargo:include={build_version_dir}"); + includes_dirs.push(build_version_dir); + } - let includes_dir = includes_dir.to_string_lossy(); - println!("cargo:include={includes_dir}"); + BindgenSpecs { + header, + includes_dirs, + } +} + +fn main() { + let bindgen_specs = get_bindgen_specs(); + for include_dir in &bindgen_specs.includes_dirs { + println!("cargo:include={include_dir}"); + } let bindings = bindgen::builder() - .clang_arg(format!("-I{includes_dir}")) - .clang_arg(format!("-I{build_version_dir}")) - .header("wrapper.h") + .clang_args( + bindgen_specs + .includes_dirs + .iter() + .map(|dir| format!("-I{dir}")), + ) + .header(bindgen_specs.header) .derive_default(true) .derive_debug(true) .prepend_enum_name(false) diff --git a/rust/mxl-sys/wrapper-with-version-h.h b/rust/mxl-sys/wrapper-with-version-h.h new file mode 100644 index 000000000..80c5254f5 --- /dev/null +++ b/rust/mxl-sys/wrapper-with-version-h.h @@ -0,0 +1,2 @@ +#include "wrapper-without-version-h.h" +#include "mxl/version.h" diff --git a/rust/mxl-sys/wrapper.h b/rust/mxl-sys/wrapper-without-version-h.h similarity index 87% rename from rust/mxl-sys/wrapper.h rename to rust/mxl-sys/wrapper-without-version-h.h index 9969e9d1c..5248f1e1f 100644 --- a/rust/mxl-sys/wrapper.h +++ b/rust/mxl-sys/wrapper-without-version-h.h @@ -5,4 +5,3 @@ #include "mxl/platform.h" #include "mxl/rational.h" #include "mxl/time.h" -#include "mxl/version.h" diff --git a/rust/mxl/Cargo.toml b/rust/mxl/Cargo.toml index 509a57899..447c49987 100644 --- a/rust/mxl/Cargo.toml +++ b/rust/mxl/Cargo.toml @@ -15,3 +15,6 @@ uuid.workspace = true [dev-dependencies] clap.workspace = true tracing-subscriber.workspace = true + +[features] +mxl-not-built = ["mxl-sys/mxl-not-built"] From c2fd34fd2c31e84524561ffb3558644cfe7fd2e9 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Thu, 31 Jul 2025 08:15:10 +0200 Subject: [PATCH 31/77] Allow instance cloning - This is useful when one wants to construct different readers and writers from different threads. Signed-off-by: Pavel Cernohorsky --- rust/mxl/src/instance.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index ca712c2f2..95861b5ec 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -14,6 +14,8 @@ pub(crate) struct InstanceContext { } // Allow sharing the context across threads and tasks freely. +// This is safe because the MXL API is supposed to be thread-safe at the +// instance level (careful, not at the reader / writer level). unsafe impl Send for InstanceContext {} unsafe impl Sync for InstanceContext {} @@ -59,6 +61,7 @@ pub(crate) fn create_flow_reader( Ok(MxlFlowReader::new(context.clone(), reader)) } +#[derive(Clone)] pub struct MxlInstance { context: Arc, } From bffed91bc8042801e51bf8cf406af602c6cc8739 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Thu, 31 Jul 2025 14:58:03 +0200 Subject: [PATCH 32/77] Implement Send for readers and writers - Will simplify use of those in async contexts, especially as long as we do not have proper async interface. Signed-off-by: Pavel Cernohorsky --- rust/mxl/src/flow_reader.rs | 4 ++++ rust/mxl/src/flow_writer.rs | 4 ++++ rust/mxl/src/grain_reader.rs | 4 ++++ rust/mxl/src/grain_writer.rs | 4 ++++ rust/mxl/src/samples_reader.rs | 4 ++++ rust/mxl/src/samples_writer.rs | 4 ++++ 6 files changed, 24 insertions(+) diff --git a/rust/mxl/src/flow_reader.rs b/rust/mxl/src/flow_reader.rs index 26991b331..3e7cc2907 100644 --- a/rust/mxl/src/flow_reader.rs +++ b/rust/mxl/src/flow_reader.rs @@ -10,6 +10,10 @@ pub struct MxlFlowReader { reader: mxl_sys::mxlFlowReader, } +/// The MXL readers and writers are not thread-safe, so we do not implement `Sync` for them, but +/// there is no reason to not implement `Send`. +unsafe impl Send for MxlFlowReader {} + pub(crate) fn get_flow_info( context: &Arc, reader: mxl_sys::mxlFlowReader, diff --git a/rust/mxl/src/flow_writer.rs b/rust/mxl/src/flow_writer.rs index ab677fcc9..fbaed1792 100644 --- a/rust/mxl/src/flow_writer.rs +++ b/rust/mxl/src/flow_writer.rs @@ -14,6 +14,10 @@ pub struct MxlFlowWriter { id: uuid::Uuid, } +/// The MXL readers and writers are not thread-safe, so we do not implement `Sync` for them, but +/// there is no reason to not implement `Send`. +unsafe impl Send for MxlFlowWriter {} + impl MxlFlowWriter { pub(crate) fn new( context: Arc, diff --git a/rust/mxl/src/grain_reader.rs b/rust/mxl/src/grain_reader.rs index a1504ecef..17341d2db 100644 --- a/rust/mxl/src/grain_reader.rs +++ b/rust/mxl/src/grain_reader.rs @@ -8,6 +8,10 @@ pub struct GrainReader { reader: mxl_sys::mxlFlowReader, } +/// The MXL readers and writers are not thread-safe, so we do not implement `Sync` for them, but +/// there is no reason to not implement `Send`. +unsafe impl Send for GrainReader {} + impl GrainReader { pub(crate) fn new(context: Arc, reader: mxl_sys::mxlFlowReader) -> Self { Self { context, reader } diff --git a/rust/mxl/src/grain_writer.rs b/rust/mxl/src/grain_writer.rs index 504f881af..9845b504d 100644 --- a/rust/mxl/src/grain_writer.rs +++ b/rust/mxl/src/grain_writer.rs @@ -8,6 +8,10 @@ pub struct GrainWriter { writer: mxl_sys::mxlFlowWriter, } +/// The MXL readers and writers are not thread-safe, so we do not implement `Sync` for them, but +/// there is no reason to not implement `Send`. +unsafe impl Send for GrainWriter {} + impl GrainWriter { pub(crate) fn new(context: Arc, writer: mxl_sys::mxlFlowWriter) -> Self { Self { context, writer } diff --git a/rust/mxl/src/samples_reader.rs b/rust/mxl/src/samples_reader.rs index e872afa60..57ad07191 100644 --- a/rust/mxl/src/samples_reader.rs +++ b/rust/mxl/src/samples_reader.rs @@ -9,6 +9,10 @@ pub struct SamplesReader { reader: mxl_sys::mxlFlowReader, } +/// The MXL readers and writers are not thread-safe, so we do not implement `Sync` for them, but +/// there is no reason to not implement `Send`. +unsafe impl Send for SamplesReader {} + impl SamplesReader { pub(crate) fn new(context: Arc, reader: mxl_sys::mxlFlowReader) -> Self { Self { context, reader } diff --git a/rust/mxl/src/samples_writer.rs b/rust/mxl/src/samples_writer.rs index 40cbe41c8..dfe9abf07 100644 --- a/rust/mxl/src/samples_writer.rs +++ b/rust/mxl/src/samples_writer.rs @@ -9,6 +9,10 @@ pub struct SamplesWriter { writer: mxl_sys::mxlFlowWriter, } +/// The MXL readers and writers are not thread-safe, so we do not implement `Sync` for them, but +/// there is no reason to not implement `Send`. +unsafe impl Send for SamplesWriter {} + impl SamplesWriter { pub(crate) fn new(context: Arc, writer: mxl_sys::mxlFlowWriter) -> Self { Self { context, writer } From 703df87368ef71f978c4ea48cb1b25f77fda6ccc Mon Sep 17 00:00:00 2001 From: Chris Chan Date: Wed, 23 Jul 2025 11:01:42 +0100 Subject: [PATCH 33/77] refactor: re-organise module structure Signed-off-by: Chris Chan --- rust/mxl/src/flow.rs | 3 ++ .../src/{flow_reader.rs => flow/reader.rs} | 9 +++--- .../src/{flow_writer.rs => flow/writer.rs} | 10 +++---- rust/mxl/src/grain.rs | 4 +++ rust/mxl/src/{grain_data.rs => grain/data.rs} | 0 .../src/{grain_reader.rs => grain/reader.rs} | 10 ++++--- .../write_access.rs} | 5 ++-- .../src/{grain_writer.rs => grain/writer.rs} | 7 +++-- rust/mxl/src/lib.rs | 28 ++++++------------- rust/mxl/src/samples.rs | 4 +++ .../src/{samples_data.rs => samples/data.rs} | 3 +- .../{samples_reader.rs => samples/reader.rs} | 8 ++++-- .../write_access.rs} | 6 ++-- .../{samples_writer.rs => samples/writer.rs} | 3 +- 14 files changed, 52 insertions(+), 48 deletions(-) rename rust/mxl/src/{flow_reader.rs => flow/reader.rs} (92%) rename rust/mxl/src/{flow_writer.rs => flow/writer.rs} (92%) create mode 100644 rust/mxl/src/grain.rs rename rust/mxl/src/{grain_data.rs => grain/data.rs} (100%) rename rust/mxl/src/{grain_reader.rs => grain/reader.rs} (93%) rename rust/mxl/src/{grain_write_access.rs => grain/write_access.rs} (96%) rename rust/mxl/src/{grain_writer.rs => grain/writer.rs} (93%) create mode 100644 rust/mxl/src/samples.rs rename rust/mxl/src/{samples_data.rs => samples/data.rs} (99%) rename rust/mxl/src/{samples_reader.rs => samples/reader.rs} (92%) rename rust/mxl/src/{samples_write_access.rs => samples/write_access.rs} (96%) rename rust/mxl/src/{samples_writer.rs => samples/writer.rs} (94%) diff --git a/rust/mxl/src/flow.rs b/rust/mxl/src/flow.rs index cfda4d845..2b81684c6 100644 --- a/rust/mxl/src/flow.rs +++ b/rust/mxl/src/flow.rs @@ -1,3 +1,6 @@ +pub mod reader; +pub mod writer; + use uuid::Uuid; use crate::{Error, Result}; diff --git a/rust/mxl/src/flow_reader.rs b/rust/mxl/src/flow/reader.rs similarity index 92% rename from rust/mxl/src/flow_reader.rs rename to rust/mxl/src/flow/reader.rs index 3e7cc2907..e63e23a2f 100644 --- a/rust/mxl/src/flow_reader.rs +++ b/rust/mxl/src/flow/reader.rs @@ -1,9 +1,10 @@ use std::sync::Arc; -use crate::flow::is_discrete_data_format; -use crate::grain_reader::GrainReader; -use crate::samples_reader::SamplesReader; -use crate::{DataFormat, Error, Result, flow::FlowInfo, instance::InstanceContext}; +use crate::{ + DataFormat, Error, GrainReader, Result, SamplesReader, + flow::{FlowInfo, is_discrete_data_format}, + instance::InstanceContext, +}; pub struct MxlFlowReader { context: Arc, diff --git a/rust/mxl/src/flow_writer.rs b/rust/mxl/src/flow/writer.rs similarity index 92% rename from rust/mxl/src/flow_writer.rs rename to rust/mxl/src/flow/writer.rs index fbaed1792..5c8d042e0 100644 --- a/rust/mxl/src/flow_writer.rs +++ b/rust/mxl/src/flow/writer.rs @@ -1,10 +1,10 @@ use std::sync::Arc; -use crate::flow::is_discrete_data_format; -use crate::grain_writer::GrainWriter; -use crate::instance::create_flow_reader; -use crate::samples_writer::SamplesWriter; -use crate::{DataFormat, Error, Result, instance::InstanceContext}; +use crate::{ + flow::is_discrete_data_format, + instance::{create_flow_reader, InstanceContext}, + DataFormat, Error, GrainWriter, Result, SamplesWriter, +}; /// Generic MXL Flow Writer, which can be further used to build either the "discrete" (grain-based /// data like video frames or meta) or "continuous" (audio samples) flow writers in MXL terminology. diff --git a/rust/mxl/src/grain.rs b/rust/mxl/src/grain.rs new file mode 100644 index 000000000..9bfae8f03 --- /dev/null +++ b/rust/mxl/src/grain.rs @@ -0,0 +1,4 @@ +pub mod data; +pub mod reader; +pub mod write_access; +pub mod writer; diff --git a/rust/mxl/src/grain_data.rs b/rust/mxl/src/grain/data.rs similarity index 100% rename from rust/mxl/src/grain_data.rs rename to rust/mxl/src/grain/data.rs diff --git a/rust/mxl/src/grain_reader.rs b/rust/mxl/src/grain/reader.rs similarity index 93% rename from rust/mxl/src/grain_reader.rs rename to rust/mxl/src/grain/reader.rs index 17341d2db..28dbaf897 100644 --- a/rust/mxl/src/grain_reader.rs +++ b/rust/mxl/src/grain/reader.rs @@ -1,7 +1,10 @@ use std::{sync::Arc, time::Duration}; -use crate::flow_reader::get_flow_info; -use crate::{Error, GrainData, Result, flow::FlowInfo, instance::InstanceContext}; +use crate::{ + flow::{reader::get_flow_info, FlowInfo}, + instance::InstanceContext, + Error, GrainData, Result, +}; pub struct GrainReader { context: Arc, @@ -49,8 +52,7 @@ impl GrainReader { } if payload_ptr.is_null() { return Err(Error::Other(format!( - "Failed to get grain payload for index {}.", - index + "Failed to get grain payload for index {index}.", ))); } break; diff --git a/rust/mxl/src/grain_write_access.rs b/rust/mxl/src/grain/write_access.rs similarity index 96% rename from rust/mxl/src/grain_write_access.rs rename to rust/mxl/src/grain/write_access.rs index 6396bb169..690f48665 100644 --- a/rust/mxl/src/grain_write_access.rs +++ b/rust/mxl/src/grain/write_access.rs @@ -1,9 +1,8 @@ -use std::marker::PhantomData; -use std::sync::Arc; +use std::{marker::PhantomData, sync::Arc}; use tracing::error; -use crate::{Error, Result, instance::InstanceContext}; +use crate::{instance::InstanceContext, Error, Result}; /// RAII grain writing session /// diff --git a/rust/mxl/src/grain_writer.rs b/rust/mxl/src/grain/writer.rs similarity index 93% rename from rust/mxl/src/grain_writer.rs rename to rust/mxl/src/grain/writer.rs index 9845b504d..5877453f5 100644 --- a/rust/mxl/src/grain_writer.rs +++ b/rust/mxl/src/grain/writer.rs @@ -1,6 +1,8 @@ use std::sync::Arc; -use crate::{Error, Result, grain_write_access::GrainWriteAccess, instance::InstanceContext}; +use super::write_access::GrainWriteAccess; + +use crate::{instance::InstanceContext, Error, Result}; /// MXL Flow Writer for discrete flows (grain-based data like video frames) pub struct GrainWriter { @@ -39,8 +41,7 @@ impl GrainWriter { if payload_ptr.is_null() { return Err(Error::Other(format!( - "Failed to open grain payload for index {}.", - index + "Failed to open grain payload for index {index}.", ))); } diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index 2cb45aad3..b2ce1e5e5 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -1,30 +1,20 @@ mod api; mod error; mod flow; -mod flow_reader; -mod flow_writer; -mod grain_data; -mod grain_reader; -mod grain_write_access; -mod grain_writer; +mod grain; mod instance; -mod samples_data; -mod samples_reader; -mod samples_write_access; -mod samples_writer; +mod samples; pub mod config; pub mod tools; pub use api::{MxlApi, load_api}; pub use error::{Error, Result}; -pub use flow::*; -pub use flow_reader::MxlFlowReader; -pub use flow_writer::MxlFlowWriter; -pub use grain_data::*; -pub use grain_reader::GrainReader; -pub use grain_write_access::GrainWriteAccess; +pub use flow::{reader::MxlFlowReader, writer::MxlFlowWriter, *}; +pub use grain::{ + data::*, reader::GrainReader, write_access::GrainWriteAccess, writer::GrainWriter, +}; pub use instance::MxlInstance; -pub use samples_data::*; -pub use samples_reader::SamplesReader; -pub use samples_write_access::SamplesWriteAccess; +pub use samples::{ + data::*, reader::SamplesReader, write_access::SamplesWriteAccess, writer::SamplesWriter, +}; diff --git a/rust/mxl/src/samples.rs b/rust/mxl/src/samples.rs new file mode 100644 index 000000000..9bfae8f03 --- /dev/null +++ b/rust/mxl/src/samples.rs @@ -0,0 +1,4 @@ +pub mod data; +pub mod reader; +pub mod write_access; +pub mod writer; diff --git a/rust/mxl/src/samples_data.rs b/rust/mxl/src/samples/data.rs similarity index 99% rename from rust/mxl/src/samples_data.rs rename to rust/mxl/src/samples/data.rs index f1d005b6a..4c0796908 100644 --- a/rust/mxl/src/samples_data.rs +++ b/rust/mxl/src/samples/data.rs @@ -1,6 +1,7 @@ -use crate::Error; use std::marker::PhantomData; +use crate::Error; + pub struct SamplesData<'a> { buffer_slice: mxl_sys::WrappedMultiBufferSlice, phantom: PhantomData<&'a ()>, diff --git a/rust/mxl/src/samples_reader.rs b/rust/mxl/src/samples/reader.rs similarity index 92% rename from rust/mxl/src/samples_reader.rs rename to rust/mxl/src/samples/reader.rs index 57ad07191..ba73e8eb8 100644 --- a/rust/mxl/src/samples_reader.rs +++ b/rust/mxl/src/samples/reader.rs @@ -1,8 +1,10 @@ use std::sync::Arc; -use crate::flow_reader::get_flow_info; -use crate::samples_data::SamplesData; -use crate::{Error, Result, flow::FlowInfo, instance::InstanceContext}; +use crate::{ + Error, Result, SamplesData, + flow::{FlowInfo, reader::get_flow_info}, + instance::InstanceContext, +}; pub struct SamplesReader { context: Arc, diff --git a/rust/mxl/src/samples_write_access.rs b/rust/mxl/src/samples/write_access.rs similarity index 96% rename from rust/mxl/src/samples_write_access.rs rename to rust/mxl/src/samples/write_access.rs index e9680950b..7a922a62a 100644 --- a/rust/mxl/src/samples_write_access.rs +++ b/rust/mxl/src/samples/write_access.rs @@ -1,10 +1,8 @@ -use std::marker::PhantomData; -use std::sync::Arc; +use std::{marker::PhantomData, sync::Arc}; use tracing::error; -use crate::Error; -use crate::instance::InstanceContext; +use crate::{instance::InstanceContext, Error}; /// RAII samples writing session /// diff --git a/rust/mxl/src/samples_writer.rs b/rust/mxl/src/samples/writer.rs similarity index 94% rename from rust/mxl/src/samples_writer.rs rename to rust/mxl/src/samples/writer.rs index dfe9abf07..939ac0d6d 100644 --- a/rust/mxl/src/samples_writer.rs +++ b/rust/mxl/src/samples/writer.rs @@ -1,7 +1,6 @@ use std::sync::Arc; -use crate::samples_write_access::SamplesWriteAccess; -use crate::{Error, Result, instance::InstanceContext}; +use crate::{instance::InstanceContext, Error, Result, SamplesWriteAccess}; /// MXL Flow Writer for continuous flows (samples-based data like audio) pub struct SamplesWriter { From f8aa6687e54cf179b917668efa63d27a6bc52d66 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Mon, 18 Aug 2025 17:30:55 +0200 Subject: [PATCH 34/77] Fix warnings reported by the current Rust compiler Signed-off-by: Pavel Cernohorsky --- rust/mxl/src/flow.rs | 2 +- rust/mxl/src/samples/reader.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/mxl/src/flow.rs b/rust/mxl/src/flow.rs index 2b81684c6..d82e0eca1 100644 --- a/rust/mxl/src/flow.rs +++ b/rust/mxl/src/flow.rs @@ -57,7 +57,7 @@ impl FlowInfo { Ok(unsafe { &self.value.__bindgen_anon_1.continuous }) } - pub fn common_flow_info(&self) -> CommonFlowInfo { + pub fn common_flow_info(&self) -> CommonFlowInfo<'_> { CommonFlowInfo(&self.value.common) } diff --git a/rust/mxl/src/samples/reader.rs b/rust/mxl/src/samples/reader.rs index ba73e8eb8..bb4727b70 100644 --- a/rust/mxl/src/samples/reader.rs +++ b/rust/mxl/src/samples/reader.rs @@ -28,7 +28,7 @@ impl SamplesReader { get_flow_info(&self.context, self.reader) } - pub fn get_samples(&self, index: u64, count: usize) -> Result { + pub fn get_samples(&self, index: u64, count: usize) -> Result> { let mut buffer_slice: mxl_sys::WrappedMultiBufferSlice = unsafe { std::mem::zeroed() }; unsafe { Error::from_status(self.context.api.mxl_flow_reader_get_samples( From fbe4482117d1bcf1fc593ea57de8a80397647503 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Mon, 18 Aug 2025 17:38:24 +0200 Subject: [PATCH 35/77] Fix warnings reported by the current Clippy Signed-off-by: Pavel Cernohorsky --- rust/mxl/src/flow/reader.rs | 10 +++++----- rust/mxl/src/flow/writer.rs | 14 +++++++------- rust/mxl/src/grain/reader.rs | 12 ++++++------ rust/mxl/src/grain/write_access.rs | 10 +++++----- rust/mxl/src/grain/writer.rs | 10 +++++----- rust/mxl/src/samples/reader.rs | 8 ++++---- rust/mxl/src/samples/write_access.rs | 10 +++++----- rust/mxl/src/samples/writer.rs | 10 +++++----- 8 files changed, 42 insertions(+), 42 deletions(-) diff --git a/rust/mxl/src/flow/reader.rs b/rust/mxl/src/flow/reader.rs index e63e23a2f..a5bfd6787 100644 --- a/rust/mxl/src/flow/reader.rs +++ b/rust/mxl/src/flow/reader.rs @@ -64,14 +64,14 @@ impl MxlFlowReader { impl Drop for MxlFlowReader { fn drop(&mut self) { - if !self.reader.is_null() { - if let Err(err) = Error::from_status(unsafe { + if !self.reader.is_null() + && let Err(err) = Error::from_status(unsafe { self.context .api .mxl_release_flow_reader(self.context.instance, self.reader) - }) { - tracing::error!("Failed to release MXL flow reader: {:?}", err); - } + }) + { + tracing::error!("Failed to release MXL flow reader: {:?}", err); } } } diff --git a/rust/mxl/src/flow/writer.rs b/rust/mxl/src/flow/writer.rs index 5c8d042e0..250578db0 100644 --- a/rust/mxl/src/flow/writer.rs +++ b/rust/mxl/src/flow/writer.rs @@ -1,9 +1,9 @@ use std::sync::Arc; use crate::{ - flow::is_discrete_data_format, - instance::{create_flow_reader, InstanceContext}, DataFormat, Error, GrainWriter, Result, SamplesWriter, + flow::is_discrete_data_format, + instance::{InstanceContext, create_flow_reader}, }; /// Generic MXL Flow Writer, which can be further used to build either the "discrete" (grain-based @@ -76,14 +76,14 @@ impl MxlFlowWriter { impl Drop for MxlFlowWriter { fn drop(&mut self) { - if !self.writer.is_null() { - if let Err(err) = Error::from_status(unsafe { + if !self.writer.is_null() + && let Err(err) = Error::from_status(unsafe { self.context .api .mxl_release_flow_writer(self.context.instance, self.writer) - }) { - tracing::error!("Failed to release MXL flow writer: {:?}", err); - } + }) + { + tracing::error!("Failed to release MXL flow writer: {:?}", err); } } } diff --git a/rust/mxl/src/grain/reader.rs b/rust/mxl/src/grain/reader.rs index 28dbaf897..227e3e8ab 100644 --- a/rust/mxl/src/grain/reader.rs +++ b/rust/mxl/src/grain/reader.rs @@ -1,9 +1,9 @@ use std::{sync::Arc, time::Duration}; use crate::{ - flow::{reader::get_flow_info, FlowInfo}, - instance::InstanceContext, Error, GrainData, Result, + flow::{FlowInfo, reader::get_flow_info}, + instance::InstanceContext, }; pub struct GrainReader { @@ -88,10 +88,10 @@ impl GrainReader { impl Drop for GrainReader { fn drop(&mut self) { - if !self.reader.is_null() { - if let Err(err) = self.destroy_inner() { - tracing::error!("Failed to release MXL flow reader (discrete): {:?}", err); - } + if !self.reader.is_null() + && let Err(err) = self.destroy_inner() + { + tracing::error!("Failed to release MXL flow reader (discrete): {:?}", err); } } } diff --git a/rust/mxl/src/grain/write_access.rs b/rust/mxl/src/grain/write_access.rs index 690f48665..04447aef4 100644 --- a/rust/mxl/src/grain/write_access.rs +++ b/rust/mxl/src/grain/write_access.rs @@ -2,7 +2,7 @@ use std::{marker::PhantomData, sync::Arc}; use tracing::error; -use crate::{instance::InstanceContext, Error, Result}; +use crate::{Error, Result, instance::InstanceContext}; /// RAII grain writing session /// @@ -85,12 +85,12 @@ impl<'a> GrainWriteAccess<'a> { impl<'a> Drop for GrainWriteAccess<'a> { fn drop(&mut self) { - if !self.committed_or_canceled { - if let Err(error) = unsafe { + if !self.committed_or_canceled + && let Err(error) = unsafe { Error::from_status(self.context.api.mxl_flow_writer_cancel_grain(self.writer)) - } { - error!("Failed to cancel grain write on drop: {:?}", error); } + { + error!("Failed to cancel grain write on drop: {:?}", error); } } } diff --git a/rust/mxl/src/grain/writer.rs b/rust/mxl/src/grain/writer.rs index 5877453f5..a2399e0b3 100644 --- a/rust/mxl/src/grain/writer.rs +++ b/rust/mxl/src/grain/writer.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use super::write_access::GrainWriteAccess; -use crate::{instance::InstanceContext, Error, Result}; +use crate::{Error, Result, instance::InstanceContext}; /// MXL Flow Writer for discrete flows (grain-based data like video frames) pub struct GrainWriter { @@ -71,10 +71,10 @@ impl GrainWriter { impl Drop for GrainWriter { fn drop(&mut self) { - if !self.writer.is_null() { - if let Err(err) = self.destroy_inner() { - tracing::error!("Failed to release MXL flow writer (discrete): {:?}", err); - } + if !self.writer.is_null() + && let Err(err) = self.destroy_inner() + { + tracing::error!("Failed to release MXL flow writer (discrete): {:?}", err); } } } diff --git a/rust/mxl/src/samples/reader.rs b/rust/mxl/src/samples/reader.rs index bb4727b70..e6f75dcdd 100644 --- a/rust/mxl/src/samples/reader.rs +++ b/rust/mxl/src/samples/reader.rs @@ -59,10 +59,10 @@ impl SamplesReader { impl Drop for SamplesReader { fn drop(&mut self) { - if !self.reader.is_null() { - if let Err(err) = self.destroy_inner() { - tracing::error!("Failed to release MXL flow reader (continuous): {:?}", err); - } + if !self.reader.is_null() + && let Err(err) = self.destroy_inner() + { + tracing::error!("Failed to release MXL flow reader (continuous): {:?}", err); } } } diff --git a/rust/mxl/src/samples/write_access.rs b/rust/mxl/src/samples/write_access.rs index 7a922a62a..bbb57ce1a 100644 --- a/rust/mxl/src/samples/write_access.rs +++ b/rust/mxl/src/samples/write_access.rs @@ -2,7 +2,7 @@ use std::{marker::PhantomData, sync::Arc}; use tracing::error; -use crate::{instance::InstanceContext, Error}; +use crate::{Error, instance::InstanceContext}; /// RAII samples writing session /// @@ -82,12 +82,12 @@ impl<'a> SamplesWriteAccess<'a> { impl<'a> Drop for SamplesWriteAccess<'a> { fn drop(&mut self) { - if !self.committed_or_canceled { - if let Err(error) = unsafe { + if !self.committed_or_canceled + && let Err(error) = unsafe { Error::from_status(self.context.api.mxl_flow_writer_cancel_samples(self.writer)) - } { - error!("Failed to cancel grain write on drop: {:?}", error); } + { + error!("Failed to cancel grain write on drop: {:?}", error); } } } diff --git a/rust/mxl/src/samples/writer.rs b/rust/mxl/src/samples/writer.rs index 939ac0d6d..4c0193f50 100644 --- a/rust/mxl/src/samples/writer.rs +++ b/rust/mxl/src/samples/writer.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use crate::{instance::InstanceContext, Error, Result, SamplesWriteAccess}; +use crate::{Error, Result, SamplesWriteAccess, instance::InstanceContext}; /// MXL Flow Writer for continuous flows (samples-based data like audio) pub struct SamplesWriter { @@ -57,10 +57,10 @@ impl SamplesWriter { impl Drop for SamplesWriter { fn drop(&mut self) { - if !self.writer.is_null() { - if let Err(err) = self.destroy_inner() { - tracing::error!("Failed to release MXL flow writer (continuous): {:?}", err); - } + if !self.writer.is_null() + && let Err(err) = self.destroy_inner() + { + tracing::error!("Failed to release MXL flow writer (continuous): {:?}", err); } } } From 242d86b1604f983a1604850825fc8312ff152560 Mon Sep 17 00:00:00 2001 From: Chris Chan Date: Fri, 1 Aug 2025 10:43:22 +0100 Subject: [PATCH 36/77] feat: first pass at github actions for rust bindings Signed-off-by: Chris Chan --- .github/workflows/rust.yml | 143 +++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 000000000..bbacba38c --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,143 @@ +name: Test the rust bindings + +on: + pull_request: + workflow_dispatch: + workflow_call: # We would like this to be called by the main job + +jobs: + dependencies: + name: Check Rust dependencies + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./rust + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-deps-${{ hashFiles('**/Cargo.toml') }} + restore-keys: ${{ runner.os }}-cargo-deps- + - name: Use Rust stable 1.88 + uses: dtolnay/rust-toolchain@1.88.0 + - name: Install cargo audit + uses: baptiste0928/cargo-install@v3 + with: + crate: cargo-audit + - name: Install cargo outdated + uses: baptiste0928/cargo-install@v3 + with: + crate: cargo-outdated + - name: Install cargo udeps + uses: baptiste0928/cargo-install@v3 + with: + crate: cargo-udeps + - name: Install cargo deny + uses: baptiste0928/cargo-install@v3 + with: + crate: cargo-deny + - name: Audit dependencies + run: cargo audit -D warnings + - name: Outdated dependencies + run: cargo outdated -d 1 -w --exit-code 1 + - name: Install nightly-2025-06-26 # Same date as 1.88 release + uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2025-06-26 + - name: Unused depedency check + run: cargo udeps --all-targets + + lint: + name: Perform Rust linting and documentation + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./rust + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-lint-${{ hashFiles('**/Cargo.toml') }} + restore-keys: ${{ runner.os }}-cargo-lint- + - name: Use Rust stable 1.88 + uses: dtolnay/rust-toolchain@1.88.0 + with: + components: rustfmt, clippy + - name: build + run: cargo build --locked + - name: Format + run: cargo fmt -- --check + - name: Docs + run: cargo doc --all-features + env: + RUSTDOCFLAGS: "-D warnings" + - name: Clippy + run: cargo clippy --all-targets -F mxl-not-built -- -D warnings + + tests: + name: Run the tests + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./rust + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-tests-${{ hashFiles('**/Cargo.toml') }} + restore-keys: ${{ runner.os }}-cargo-tests- + - name: Use Rust stable 1.88 + uses: dtolnay/rust-toolchain@1.88.0 + with: + components: rustfmt, clippy + - name: build + run: cargo build --locked -F mxl-not-built + - name: Test + run: cargo test --locked -F mxl-not-built + - name: Coverage + run: > + cargo llvm-cov --ignore-filename-regex "build.rs|ffi.rs|(.*)_test.rs" + --lcov --output-path lcov.info + - name: Report Coverage + uses: romeovs/lcov-reporter-action@v0.4.0 + if: ${{ github.event_name == 'pull_request' }} + with: + lcov-file: ./lcov.info + github-token: ${{ secrets.GITHUB_TOKEN }} + delete-old-comments: true From 3421135043b90596d406152069acee439b114e89 Mon Sep 17 00:00:00 2001 From: Chris Chan Date: Thu, 14 Aug 2025 00:46:55 +0100 Subject: [PATCH 37/77] feat: add build options to build neither tests nor tools Signed-off-by: Chris Chan --- CMakeLists.txt | 7 ++++- lib/CMakeLists.txt | 4 ++- rust/flake.lock | 0 rust/flake.nix | 70 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 rust/flake.lock create mode 100644 rust/flake.nix diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a265dbe5..638e5b35d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,9 @@ if(ccache_executable) set(CMAKE_CXX_COMPILER_LAUNCHER ${ccache_executable}) endif() +option(BUILD_TESTS "Build the tests" OFF) +option(BUILD_TOOLS "Build the tools" ON) + # Enable testing for the project enable_testing() @@ -54,8 +57,10 @@ if(APPLE) endif() add_subdirectory(lib) -add_subdirectory(tools) add_subdirectory(utils) +if (BUILD_TOOLS) + add_subdirectory(tools) +endif() find_package(Doxygen) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index d9bbc6c7d..71b739cf5 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -81,7 +81,9 @@ target_link_libraries(mxl # Alias mxl to mxl::mxl so that this library can be used # in lieu of a module from the local source tree add_library(${PROJECT_NAME}::mxl ALIAS mxl) -add_subdirectory(tests) +if (BUILD_TESTS) + add_subdirectory(tests) +endif() # Install targets install(TARGETS mxl EXPORT ${PROJECT_NAME}-targets diff --git a/rust/flake.lock b/rust/flake.lock new file mode 100644 index 000000000..e69de29bb diff --git a/rust/flake.nix b/rust/flake.nix new file mode 100644 index 000000000..a219b532b --- /dev/null +++ b/rust/flake.nix @@ -0,0 +1,70 @@ +{ + description = "Flake for MXL dev"; + + inputs = { + nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/*.tar.gz"; + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = { + self, + nixpkgs, + rust-overlay + }: let + overlays = [ + (import rust-overlay) + (self: super: { + rustStable = super.rust-bin.stable."1.88.0".default; + rustNightly = super.rust-bin.nightly."2025-06-26".default; + }) + ]; + + allSystems = [ + "x86_64-linux" # 64-bit Intel/AMD Linux + "aarch64-linux" # 64-bit ARM Linux + "x86_64-darwin" # 64-bit Intel macOS + "aarch64-darwin" # 64-bit ARM macOS + ]; + + forAllSystems = f: + nixpkgs.lib.genAttrs allSystems (system: + f { + pkgs = import nixpkgs { + inherit overlays system; + }; + } + ); + in { + devShells = forAllSystems ({pkgs}: { + default = pkgs.mkShell { + LIBCLANG_PATH = pkgs.lib.makeLibraryPath [pkgs.llvmPackages_latest.libclang.lib]; + packages = + (with pkgs; [ + rustStable + rust-analyzer + clang + cmake + pkg-config + ]); + }; + } + ); + nightly = forAllSystems ({pkgs}: { + default = pkgs.mkShell { + LIBCLANG_PATH = pkgs.lib.makeLibraryPath [pkgs.llvmPackages_latest.libclang.lib]; + packages = + (with pkgs; [ + rustNightly + rust-analyzer + clang + cmake + pkg-config + ]); + }; + } + ); + }; +} From d066d5683030a4e20d214781efdf36bd98552d73 Mon Sep 17 00:00:00 2001 From: Chris Chan Date: Thu, 14 Aug 2025 00:50:16 +0100 Subject: [PATCH 38/77] feat: ignore generated mxl version header file Signed-off-by: Chris Chan --- rust/mxl-sys/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 rust/mxl-sys/.gitignore diff --git a/rust/mxl-sys/.gitignore b/rust/mxl-sys/.gitignore new file mode 100644 index 000000000..8d6e34004 --- /dev/null +++ b/rust/mxl-sys/.gitignore @@ -0,0 +1 @@ +mxl/version.h From 7b6207c90fb9396bdf3efa73a4e2240fbb8dd6f7 Mon Sep 17 00:00:00 2001 From: Chris Chan Date: Thu, 14 Aug 2025 14:22:20 +0100 Subject: [PATCH 39/77] feat: add options to not build tools nor tests Signed-off-by: Chris Chan --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 638e5b35d..20b59c86f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,9 @@ else() string(APPEND mxl_VERSION ".0") endif() +option(BUILD_TESTS "Build the tests" ON) +option(BUILD_TOOLS "Build the tools" ON) + project(mxl VERSION ${mxl_VERSION} LANGUAGES CXX C @@ -37,9 +40,6 @@ if(ccache_executable) set(CMAKE_CXX_COMPILER_LAUNCHER ${ccache_executable}) endif() -option(BUILD_TESTS "Build the tests" OFF) -option(BUILD_TOOLS "Build the tools" ON) - # Enable testing for the project enable_testing() From c90a5d18616e2dc9b26256426a09e0fc3b77a09e Mon Sep 17 00:00:00 2001 From: Chris Chan Date: Thu, 14 Aug 2025 15:18:11 +0100 Subject: [PATCH 40/77] feat: use `cmake` in `build.rs` for `mxl-sys` Signed-off-by: Chris Chan --- rust/Cargo.lock | 19 +++++++++++++++++++ rust/mxl-sys/Cargo.toml | 1 + rust/mxl-sys/build.rs | 29 +++++++++++++++++++++++++++-- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 6a731e570..eda5e2d4f 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -60,6 +60,15 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "cc" +version = "1.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" +dependencies = [ + "shlex", +] + [[package]] name = "cexpr" version = "0.6.0" @@ -124,6 +133,15 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "dlopen2" version = "0.8.0" @@ -251,6 +269,7 @@ name = "mxl-sys" version = "0.1.0" dependencies = [ "bindgen", + "cmake", ] [[package]] diff --git a/rust/mxl-sys/Cargo.toml b/rust/mxl-sys/Cargo.toml index 6bbb0ffdd..0e4fa65a9 100644 --- a/rust/mxl-sys/Cargo.toml +++ b/rust/mxl-sys/Cargo.toml @@ -8,6 +8,7 @@ version.workspace = true [build-dependencies] bindgen.workspace = true +cmake = "0.1.54" [features] mxl-not-built = [] diff --git a/rust/mxl-sys/build.rs b/rust/mxl-sys/build.rs index 0d76aff09..625302cc4 100644 --- a/rust/mxl-sys/build.rs +++ b/rust/mxl-sys/build.rs @@ -1,4 +1,5 @@ use std::env; +use std::fs; use std::path::PathBuf; #[cfg(debug_assertions)] @@ -27,8 +28,7 @@ fn get_bindgen_specs() -> BindgenSpecs { .to_string_lossy() .to_string(), ]; - #[cfg(not(feature = "mxl-not-built"))] - { + if cfg!(not(feature = "mxl-not-built")) { let build_dir = repo_root.join("build").join(BUILD_VARIANT); let build_version_dir = build_dir .join("lib") @@ -51,6 +51,31 @@ fn main() { println!("cargo:include={include_dir}"); } + if cfg!(not(feature = "mxl-not-built")) { + // TODO: figure out when this has to be rebuilt + let mxl_version_out_path = + PathBuf::from(&env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set")) + .join("mxl"); + if !fs::exists(&mxl_version_out_path).expect("Error checking if out path exists") { + fs::create_dir(&mxl_version_out_path).expect("Failed to create out path"); + } + let out_path = mxl_version_out_path.join("version.h"); + println!("cargo:rerun-if-changed={}", out_path.display()); + + let dst = cmake::Config::new("../../") + .define("BUILD_TESTS", "OFF") + .define("BUILD_TOOLS", "OFF") + .build(); + + let mxl_version_location = dst.join("include").join("mxl").join("version.h"); + assert!(matches!(std::fs::exists(&mxl_version_location), Ok(true))); + + fs::copy(&mxl_version_location, &out_path).expect("Could copy mxl version"); + + println!("cargo:rustc-link-search={}", dst.join("lib64").display()); + println!("cargo:rustc-link-lib=mxl"); + } + let bindings = bindgen::builder() .clang_args( bindgen_specs From 27cacdf310e231b848596de80aec9f07924c4226 Mon Sep 17 00:00:00 2001 From: Chris Chan Date: Thu, 14 Aug 2025 15:23:15 +0100 Subject: [PATCH 41/77] refactor: move code for compiling `mxl` with cmake Signed-off-by: Chris Chan --- rust/mxl-sys/build.rs | 47 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/rust/mxl-sys/build.rs b/rust/mxl-sys/build.rs index 625302cc4..ca0eea5dd 100644 --- a/rust/mxl-sys/build.rs +++ b/rust/mxl-sys/build.rs @@ -37,32 +37,18 @@ fn get_bindgen_specs() -> BindgenSpecs { .to_string(); includes_dirs.push(build_version_dir); - } - - BindgenSpecs { - header, - includes_dirs, - } -} - -fn main() { - let bindgen_specs = get_bindgen_specs(); - for include_dir in &bindgen_specs.includes_dirs { - println!("cargo:include={include_dir}"); - } - if cfg!(not(feature = "mxl-not-built")) { - // TODO: figure out when this has to be rebuilt - let mxl_version_out_path = - PathBuf::from(&env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set")) - .join("mxl"); - if !fs::exists(&mxl_version_out_path).expect("Error checking if out path exists") { - fs::create_dir(&mxl_version_out_path).expect("Failed to create out path"); + let mxl_version_out_path = manifest_dir.join("mxl"); + if !fs::exists(&mxl_version_out_path) + .expect("Error checking if out path for version header file exists") + { + fs::create_dir(&mxl_version_out_path) + .expect("Failed to create out path for version header file"); } - let out_path = mxl_version_out_path.join("version.h"); - println!("cargo:rerun-if-changed={}", out_path.display()); + let mxl_version_header = mxl_version_out_path.join("version.h"); + println!("cargo:rerun-if-changed={}", mxl_version_header.display()); - let dst = cmake::Config::new("../../") + let dst = cmake::Config::new(repo_root) .define("BUILD_TESTS", "OFF") .define("BUILD_TOOLS", "OFF") .build(); @@ -70,12 +56,25 @@ fn main() { let mxl_version_location = dst.join("include").join("mxl").join("version.h"); assert!(matches!(std::fs::exists(&mxl_version_location), Ok(true))); - fs::copy(&mxl_version_location, &out_path).expect("Could copy mxl version"); + fs::copy(&mxl_version_location, &mxl_version_header) + .expect("Could copy mxl version header"); println!("cargo:rustc-link-search={}", dst.join("lib64").display()); println!("cargo:rustc-link-lib=mxl"); } + BindgenSpecs { + header, + includes_dirs, + } +} + +fn main() { + let bindgen_specs = get_bindgen_specs(); + for include_dir in &bindgen_specs.includes_dirs { + println!("cargo:include={include_dir}"); + } + let bindings = bindgen::builder() .clang_args( bindgen_specs From 71fd4c25e4c225a4ffb24ad5a8e1fd969f68a6eb Mon Sep 17 00:00:00 2001 From: Chris Chan Date: Fri, 15 Aug 2025 09:42:46 +0100 Subject: [PATCH 42/77] fix: update `build.rs` to build into build dir Signed-off-by: Chris Chan --- rust/mxl-sys/build.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/mxl-sys/build.rs b/rust/mxl-sys/build.rs index ca0eea5dd..a4220109e 100644 --- a/rust/mxl-sys/build.rs +++ b/rust/mxl-sys/build.rs @@ -49,8 +49,10 @@ fn get_bindgen_specs() -> BindgenSpecs { println!("cargo:rerun-if-changed={}", mxl_version_header.display()); let dst = cmake::Config::new(repo_root) + .out_dir("build_dir") .define("BUILD_TESTS", "OFF") .define("BUILD_TOOLS", "OFF") + .configure_arg(format!("--preset={BUILD_VARIANT}")) .build(); let mxl_version_location = dst.join("include").join("mxl").join("version.h"); From afa0df3a47a61eb22fa8defb872a34e326e26567 Mon Sep 17 00:00:00 2001 From: Chris Chan Date: Fri, 15 Aug 2025 09:49:16 +0100 Subject: [PATCH 43/77] fix: supply build_dir correctly as path Signed-off-by: Chris Chan --- rust/mxl-sys/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/mxl-sys/build.rs b/rust/mxl-sys/build.rs index a4220109e..90ab1b811 100644 --- a/rust/mxl-sys/build.rs +++ b/rust/mxl-sys/build.rs @@ -49,7 +49,7 @@ fn get_bindgen_specs() -> BindgenSpecs { println!("cargo:rerun-if-changed={}", mxl_version_header.display()); let dst = cmake::Config::new(repo_root) - .out_dir("build_dir") + .out_dir(build_dir) .define("BUILD_TESTS", "OFF") .define("BUILD_TOOLS", "OFF") .configure_arg(format!("--preset={BUILD_VARIANT}")) From b97d83ea178bf19253f37f534fce39ead782dfe8 Mon Sep 17 00:00:00 2001 From: Chris Chan Date: Fri, 15 Aug 2025 14:48:49 +0100 Subject: [PATCH 44/77] fix: building the shared library for the rust bindings Signed-off-by: Chris Chan --- rust/mxl-sys/build.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/mxl-sys/build.rs b/rust/mxl-sys/build.rs index 90ab1b811..390ce9707 100644 --- a/rust/mxl-sys/build.rs +++ b/rust/mxl-sys/build.rs @@ -47,12 +47,14 @@ fn get_bindgen_specs() -> BindgenSpecs { } let mxl_version_header = mxl_version_out_path.join("version.h"); println!("cargo:rerun-if-changed={}", mxl_version_header.display()); + // TODO: re-run on build_dir changing? let dst = cmake::Config::new(repo_root) .out_dir(build_dir) + .generator("Unix Makefiles") .define("BUILD_TESTS", "OFF") .define("BUILD_TOOLS", "OFF") - .configure_arg(format!("--preset={BUILD_VARIANT}")) + // .configure_arg(format!("--preset={BUILD_VARIANT}")) .build(); let mxl_version_location = dst.join("include").join("mxl").join("version.h"); From 09e078536924c30d11b2167deadda04b2632d8a4 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 25 Sep 2025 12:06:31 +0000 Subject: [PATCH 45/77] devcontainer: add rust tooling to container - Add a new forlder for scripts that can be used to avoid duplicating code across Dockerfiles. - Add a script to install rust and related tools Signed-off-by: Pedro Ferreira --- .devcontainer/Dockerfile | 3 +- .devcontainer/Dockerfile.almalinux | 3 +- .devcontainer/Dockerfile.amazonlinux | 3 +- .devcontainer/Dockerfile.debiantrixie | 3 +- .devcontainer/Dockerfile.ubuntu-legacy | 3 +- .../scripts/common/rust/install-rust.sh | 29 +++++++++++++++++++ 6 files changed, 39 insertions(+), 5 deletions(-) create mode 100755 .devcontainer/scripts/common/rust/install-rust.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b4af36042..8437333da 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -86,4 +86,5 @@ RUN git clone https://github.com/microsoft/vcpkg \ ENV VCPKG_ROOT="/home/${USERNAME}/vcpkg" -RUN rustup default 1.88.0 +COPY ./scripts/ /tmp/scripts/ +RUN /tmp/scripts/common/rust/install-rust.sh diff --git a/.devcontainer/Dockerfile.almalinux b/.devcontainer/Dockerfile.almalinux index 4b1b378e8..19c274f7a 100644 --- a/.devcontainer/Dockerfile.almalinux +++ b/.devcontainer/Dockerfile.almalinux @@ -73,4 +73,5 @@ RUN git clone https://github.com/microsoft/vcpkg \ ENV VCPKG_ROOT=/home/$USERNAME/vcpkg -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=1.88.0 +COPY ./scripts/ /tmp/scripts/ +RUN /tmp/scripts/common/rust/install-rust.sh diff --git a/.devcontainer/Dockerfile.amazonlinux b/.devcontainer/Dockerfile.amazonlinux index e5ce400b8..134763dc4 100644 --- a/.devcontainer/Dockerfile.amazonlinux +++ b/.devcontainer/Dockerfile.amazonlinux @@ -66,4 +66,5 @@ RUN git clone https://github.com/microsoft/vcpkg \ ENV VCPKG_ROOT=/home/$USERNAME/vcpkg -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=1.88.0 +COPY ./scripts/ /tmp/scripts/ +RUN /tmp/scripts/common/rust/install-rust.sh diff --git a/.devcontainer/Dockerfile.debiantrixie b/.devcontainer/Dockerfile.debiantrixie index 923648c53..ba0d76d93 100644 --- a/.devcontainer/Dockerfile.debiantrixie +++ b/.devcontainer/Dockerfile.debiantrixie @@ -81,4 +81,5 @@ RUN git clone https://github.com/microsoft/vcpkg \ ENV VCPKG_ROOT="/home/${USERNAME}/vcpkg" -RUN rustup default 1.88.0 +COPY ./scripts/ /tmp/scripts/ +RUN /tmp/scripts/common/rust/install-rust.sh diff --git a/.devcontainer/Dockerfile.ubuntu-legacy b/.devcontainer/Dockerfile.ubuntu-legacy index 69ba26e8b..4ca0677c3 100644 --- a/.devcontainer/Dockerfile.ubuntu-legacy +++ b/.devcontainer/Dockerfile.ubuntu-legacy @@ -100,4 +100,5 @@ ENV VCPKG_ROOT=/home/$USERNAME/vcpkg COPY register-clang-version.sh /home/$USERNAME/register-clang-version.sh RUN sudo /home/$USERNAME/register-clang-version.sh ${CLANG_VERSION} 100 && rm /home/$USERNAME/register-clang-version.sh -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=1.88.0 +COPY ./scripts/ /tmp/scripts/ +RUN /tmp/scripts/common/rust/install-rust.sh diff --git a/.devcontainer/scripts/common/rust/install-rust.sh b/.devcontainer/scripts/common/rust/install-rust.sh new file mode 100755 index 000000000..ae7696db1 --- /dev/null +++ b/.devcontainer/scripts/common/rust/install-rust.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +RUST_VERSION=1.88.0 + +if command -v rustup >/dev/null 2>&1; then + rustup default $RUST_VERSION +else + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=$RUST_VERSION + . "$HOME/.cargo/env" +fi + +# Install cargo binstall +curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash + +cargo binstall cargo-audit --locked +cargo binstall cargo-outdated --locked + +# udeps requires the nightly compiler, so using machete (at least for now) +# cargo binstall cargo-udeps --locked +cargo binstall cargo-machete --locked + +cargo binstall cargo-deny --locked +cargo binstall cargo-audit --locked +cargo binstall cargo-nextest --locked From d41dfe0a46a67a61c1f8ea1c34bc40276b25dda9 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 25 Sep 2025 12:07:07 +0000 Subject: [PATCH 46/77] cmake: add an option to disable building the docs Signed-off-by: Pedro Ferreira --- CMakeLists.txt | 55 ++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 20b59c86f..1efaf7716 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ else() string(APPEND mxl_VERSION ".0") endif() +option(BUILD_DOCS "Build the docs" ON) option(BUILD_TESTS "Build the tests" ON) option(BUILD_TOOLS "Build the tools" ON) @@ -62,39 +63,41 @@ if (BUILD_TOOLS) add_subdirectory(tools) endif() -find_package(Doxygen) +if(BUILD_DOCS) + find_package(Doxygen) -if(DOXYGEN_FOUND) - include(FetchContent) + if(DOXYGEN_FOUND) + include(FetchContent) - FetchContent_Declare( - doxygen-awesome-css - URL https://github.com/jothepro/doxygen-awesome-css/archive/refs/heads/main.zip - ) - FetchContent_MakeAvailable(doxygen-awesome-css) + FetchContent_Declare( + doxygen-awesome-css + URL https://github.com/jothepro/doxygen-awesome-css/archive/refs/heads/main.zip + ) + FetchContent_MakeAvailable(doxygen-awesome-css) - # Save the location the files were cloned into - # This allows us to get the path to doxygen-awesome.css - FetchContent_GetProperties(doxygen-awesome-css SOURCE_DIR AWESOME_CSS_DIR) + # Save the location the files were cloned into + # This allows us to get the path to doxygen-awesome.css + FetchContent_GetProperties(doxygen-awesome-css SOURCE_DIR AWESOME_CSS_DIR) - # Generate the Doxyfile - set(DOXYFILE_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in) - set(DOXYFILE_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile) - configure_file(${DOXYFILE_IN} ${DOXYFILE_OUT} @ONLY) + # Generate the Doxyfile + set(DOXYFILE_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in) + set(DOXYFILE_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile) + configure_file(${DOXYFILE_IN} ${DOXYFILE_OUT} @ONLY) - set(DOXYGEN_OUTPUT_DIR "${CMAKE_BINARY_DIR}/docs") + set(DOXYGEN_OUTPUT_DIR "${CMAKE_BINARY_DIR}/docs") - add_custom_target(doc - COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_OUT} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Generating API documentation with Doxygen" - BYPRODUCTS "${DOXYGEN_OUTPUT_DIR}/html/index.html" - VERBATIM - ) + add_custom_target(doc + COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_OUT} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating API documentation with Doxygen" + BYPRODUCTS "${DOXYGEN_OUTPUT_DIR}/html/index.html" + VERBATIM + ) - install(DIRECTORY "${CMAKE_BINARY_DIR}/docs/html" - DESTINATION share/doc/mxl - FILES_MATCHING PATTERN "*") + install(DIRECTORY "${CMAKE_BINARY_DIR}/docs/html" + DESTINATION share/doc/mxl + FILES_MATCHING PATTERN "*") + endif() endif() if(EXISTS "/etc/os-release") From 7a566ec2f1dc4a645a3f740821eba236ab45b110 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 25 Sep 2025 12:09:00 +0000 Subject: [PATCH 47/77] build.rs: use CMake presets Signed-off-by: Pedro Ferreira --- rust/mxl-sys/build.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/rust/mxl-sys/build.rs b/rust/mxl-sys/build.rs index 390ce9707..5ea02f470 100644 --- a/rust/mxl-sys/build.rs +++ b/rust/mxl-sys/build.rs @@ -48,13 +48,19 @@ fn get_bindgen_specs() -> BindgenSpecs { let mxl_version_header = mxl_version_out_path.join("version.h"); println!("cargo:rerun-if-changed={}", mxl_version_header.display()); // TODO: re-run on build_dir changing? + // TODO: re-run on any changes in lib + + let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); let dst = cmake::Config::new(repo_root) - .out_dir(build_dir) - .generator("Unix Makefiles") + .generator("Ninja") + .configure_arg("--preset") + .configure_arg(BUILD_VARIANT) + .configure_arg("-B") + .configure_arg(out_dir.join("build")) + .define("BUILD_DOCS", "OFF") .define("BUILD_TESTS", "OFF") .define("BUILD_TOOLS", "OFF") - // .configure_arg(format!("--preset={BUILD_VARIANT}")) .build(); let mxl_version_location = dst.join("include").join("mxl").join("version.h"); From f016b243ba0f399866ba890cee81a62c1f65d876 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 25 Sep 2025 12:46:49 +0000 Subject: [PATCH 48/77] build.rs: simplify binding generation Assume that mxl is always built in the target directory. Signed-off-by: Pedro Ferreira --- rust/mxl-sys/build.rs | 33 ++++++--------------------------- rust/mxl/src/config.rs | 7 +++---- 2 files changed, 9 insertions(+), 31 deletions(-) diff --git a/rust/mxl-sys/build.rs b/rust/mxl-sys/build.rs index 5ea02f470..2eaf1d9f3 100644 --- a/rust/mxl-sys/build.rs +++ b/rust/mxl-sys/build.rs @@ -1,5 +1,4 @@ use std::env; -use std::fs; use std::path::PathBuf; #[cfg(debug_assertions)] @@ -29,28 +28,14 @@ fn get_bindgen_specs() -> BindgenSpecs { .to_string(), ]; if cfg!(not(feature = "mxl-not-built")) { - let build_dir = repo_root.join("build").join(BUILD_VARIANT); - let build_version_dir = build_dir - .join("lib") - .join("include") - .to_string_lossy() - .to_string(); + let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + let build_version_dir = out_dir.join("include").to_string_lossy().to_string(); includes_dirs.push(build_version_dir); - let mxl_version_out_path = manifest_dir.join("mxl"); - if !fs::exists(&mxl_version_out_path) - .expect("Error checking if out path for version header file exists") - { - fs::create_dir(&mxl_version_out_path) - .expect("Failed to create out path for version header file"); - } - let mxl_version_header = mxl_version_out_path.join("version.h"); - println!("cargo:rerun-if-changed={}", mxl_version_header.display()); - // TODO: re-run on build_dir changing? - // TODO: re-run on any changes in lib - - let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + // Rebuild if any file in lib/ changes + let lib_root = repo_root.join("lib"); + println!("cargo:rerun-if-changed={}", lib_root.display()); let dst = cmake::Config::new(repo_root) .generator("Ninja") @@ -63,13 +48,7 @@ fn get_bindgen_specs() -> BindgenSpecs { .define("BUILD_TOOLS", "OFF") .build(); - let mxl_version_location = dst.join("include").join("mxl").join("version.h"); - assert!(matches!(std::fs::exists(&mxl_version_location), Ok(true))); - - fs::copy(&mxl_version_location, &mxl_version_header) - .expect("Could copy mxl version header"); - - println!("cargo:rustc-link-search={}", dst.join("lib64").display()); + println!("cargo:rustc-link-search={}", dst.join("lib").display()); println!("cargo:rustc-link-lib=mxl"); } diff --git a/rust/mxl/src/config.rs b/rust/mxl/src/config.rs index f89891ae5..0b333e8b5 100644 --- a/rust/mxl/src/config.rs +++ b/rust/mxl/src/config.rs @@ -3,10 +3,9 @@ use std::str::FromStr; include!(concat!(env!("OUT_DIR"), "/constants.rs")); pub fn get_mxf_so_path() -> std::path::PathBuf { - std::path::PathBuf::from_str(MXL_BUILD_DIR) - .expect("build error: 'MXL_BUILD_DIR' is invalid") - .join("lib") - .join("libmxl.so") + // The mxl-sys build script ensures that the build directory is in the library path + // so we can just return the library name here. + "libmxl.so".into() } pub fn get_mxl_repo_root() -> std::path::PathBuf { From 35131fe4dee21addd9a798f430d67469d1e0b8f6 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 25 Sep 2025 16:03:31 +0000 Subject: [PATCH 49/77] rust: update to 1.90.0 Signed-off-by: Pedro Ferreira --- .devcontainer/scripts/common/rust/install-rust.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/scripts/common/rust/install-rust.sh b/.devcontainer/scripts/common/rust/install-rust.sh index ae7696db1..fbab87790 100755 --- a/.devcontainer/scripts/common/rust/install-rust.sh +++ b/.devcontainer/scripts/common/rust/install-rust.sh @@ -5,9 +5,9 @@ set -eu -RUST_VERSION=1.88.0 +RUST_VERSION=1.90.0 -if command -v rustup >/dev/null 2>&1; then +if command -v rustup > /dev/null 2>&1; then rustup default $RUST_VERSION else curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=$RUST_VERSION From 4d88a779f18e29336c59bd31e5bd8b8166c7d03b Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 25 Sep 2025 18:52:40 +0000 Subject: [PATCH 50/77] rust: correct typo in get_mxl_so_path Signed-off-by: Pedro Ferreira --- rust/mxl/examples/flow-reader.rs | 4 ++-- rust/mxl/examples/flow-writer.rs | 4 ++-- rust/mxl/src/config.rs | 2 +- rust/mxl/tests/basic_tests.rs | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rust/mxl/examples/flow-reader.rs b/rust/mxl/examples/flow-reader.rs index 98bd2a2c0..5fce97946 100644 --- a/rust/mxl/examples/flow-reader.rs +++ b/rust/mxl/examples/flow-reader.rs @@ -3,7 +3,7 @@ mod common; use std::time::Duration; use clap::Parser; -use mxl::config::get_mxf_so_path; +use mxl::config::get_mxl_so_path; use tracing::{info, warn}; const READ_TIMEOUT: Duration = Duration::from_secs(5); @@ -30,7 +30,7 @@ fn main() -> Result<(), mxl::Error> { common::setup_logging(); let opts: Opts = Opts::parse(); - let mxl_api = mxl::load_api(get_mxf_so_path())?; + let mxl_api = mxl::load_api(get_mxl_so_path())?; let mxl_instance = mxl::MxlInstance::new(mxl_api, &opts.mxl_domain, "")?; let reader = mxl_instance.create_flow_reader(&opts.flow_id)?; let flow_info = reader.get_info()?; diff --git a/rust/mxl/examples/flow-writer.rs b/rust/mxl/examples/flow-writer.rs index 7fe5ba138..b91bfc3a8 100644 --- a/rust/mxl/examples/flow-writer.rs +++ b/rust/mxl/examples/flow-writer.rs @@ -3,7 +3,7 @@ mod common; use clap::Parser; use tracing::info; -use mxl::config::get_mxf_so_path; +use mxl::config::get_mxl_so_path; #[derive(Debug, Parser)] #[command(version = clap::crate_version!(), author = clap::crate_authors!())] @@ -30,7 +30,7 @@ fn main() -> Result<(), mxl::Error> { common::setup_logging(); let opts: Opts = Opts::parse(); - let mxl_api = mxl::load_api(get_mxf_so_path())?; + let mxl_api = mxl::load_api(get_mxl_so_path())?; let mxl_instance = mxl::MxlInstance::new(mxl_api, &opts.mxl_domain, "")?; let flow_def = mxl::tools::read_file(opts.flow_config_file.as_str()).map_err(|error| { mxl::Error::Other(format!( diff --git a/rust/mxl/src/config.rs b/rust/mxl/src/config.rs index 0b333e8b5..d40018708 100644 --- a/rust/mxl/src/config.rs +++ b/rust/mxl/src/config.rs @@ -2,7 +2,7 @@ use std::str::FromStr; include!(concat!(env!("OUT_DIR"), "/constants.rs")); -pub fn get_mxf_so_path() -> std::path::PathBuf { +pub fn get_mxl_so_path() -> std::path::PathBuf { // The mxl-sys build script ensures that the build directory is in the library path // so we can just return the library name here. "libmxl.so".into() diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index 7242077ca..fb5295bf0 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -4,7 +4,7 @@ /// change in the future. For now, feel free to just edit the path to your library. use std::time::Duration; -use mxl::{MxlInstance, OwnedGrainData, OwnedSamplesData, config::get_mxf_so_path}; +use mxl::{MxlInstance, OwnedGrainData, OwnedSamplesData, config::get_mxl_so_path}; use tracing::info; static LOG_ONCE: std::sync::Once = std::sync::Once::new(); @@ -32,7 +32,7 @@ fn setup_test(test: &str) -> mxl::MxlInstance { .init(); }); - let mxl_api = mxl::load_api(get_mxf_so_path()).unwrap(); + let mxl_api = mxl::load_api(get_mxl_so_path()).unwrap(); let domain = setup_empty_domain(test); mxl::MxlInstance::new(mxl_api, &domain, "").unwrap() } From d28e9673a22eb3fd92d46b7c1d91ce991362ad77 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 24 Jun 2025 12:02:34 +0000 Subject: [PATCH 51/77] doc: fix warnings in doc generation for bindings Signed-off-by: Pedro Ferreira --- rust/mxl-sys/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/mxl-sys/src/lib.rs b/rust/mxl-sys/src/lib.rs index 666331b0d..2e4dc43dd 100644 --- a/rust/mxl-sys/src/lib.rs +++ b/rust/mxl-sys/src/lib.rs @@ -2,6 +2,8 @@ #![allow(non_camel_case_types)] #![allow(non_snake_case)] #![allow(missing_docs)] +#![allow(rustdoc::broken_intra_doc_links)] +#![allow(rustdoc::invalid_html_tags)] // Suppress expected warnings from bindgen-generated code. // See https://github.com/rust-lang/rust-bindgen/issues/1651. #![allow(deref_nullptr)] From 3eb999d21c79e04af5cdeee743a8b646f35b9ec8 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 25 Sep 2025 19:06:43 +0000 Subject: [PATCH 52/77] devcontainer: add rust analyzer plugin Signed-off-by: Pedro Ferreira --- .devcontainer/almalinux/devcontainer.json | 5 +++-- .devcontainer/amazonlinux/devcontainer.json | 5 +++-- .devcontainer/debiantrixie/devcontainer.json | 5 +++-- .devcontainer/ubuntu20/devcontainer.json | 5 +++-- .devcontainer/ubuntu22/devcontainer.json | 5 +++-- .devcontainer/ubuntu24/devcontainer.json | 5 +++-- .devcontainer/ubuntu25/devcontainer.json | 5 +++-- 7 files changed, 21 insertions(+), 14 deletions(-) diff --git a/.devcontainer/almalinux/devcontainer.json b/.devcontainer/almalinux/devcontainer.json index c80d44b2d..252bbf680 100644 --- a/.devcontainer/almalinux/devcontainer.json +++ b/.devcontainer/almalinux/devcontainer.json @@ -28,7 +28,8 @@ "matepek.vscode-catch2-test-adapter", "llvm-vs-code-extensions.vscode-clangd", "xaver.clang-format", - "bierner.markdown-mermaid" + "bierner.markdown-mermaid", + "rust-lang.rust-analyzer" ], "settings": { "C_Cpp.intelliSenseEngine": "disabled", @@ -50,4 +51,4 @@ "gpu": "optional" }, "remoteUser": "devcontainer" -} \ No newline at end of file +} diff --git a/.devcontainer/amazonlinux/devcontainer.json b/.devcontainer/amazonlinux/devcontainer.json index 7d456894e..2610a3e6e 100644 --- a/.devcontainer/amazonlinux/devcontainer.json +++ b/.devcontainer/amazonlinux/devcontainer.json @@ -32,7 +32,8 @@ "matepek.vscode-catch2-test-adapter", "llvm-vs-code-extensions.vscode-clangd", "xaver.clang-format", - "bierner.markdown-mermaid" + "bierner.markdown-mermaid", + "rust-lang.rust-analyzer" ], "settings": { "C_Cpp.intelliSenseEngine": "disabled", @@ -51,4 +52,4 @@ "DISPLAY": "${localEnv:DISPLAY}" }, "remoteUser": "devcontainer" -} \ No newline at end of file +} diff --git a/.devcontainer/debiantrixie/devcontainer.json b/.devcontainer/debiantrixie/devcontainer.json index bb5b17b3b..684157d1b 100644 --- a/.devcontainer/debiantrixie/devcontainer.json +++ b/.devcontainer/debiantrixie/devcontainer.json @@ -28,7 +28,8 @@ "matepek.vscode-catch2-test-adapter", "llvm-vs-code-extensions.vscode-clangd", "xaver.clang-format", - "bierner.markdown-mermaid" + "bierner.markdown-mermaid", + "rust-lang.rust-analyzer" ], "settings": { "C_Cpp.intelliSenseEngine": "disabled", @@ -50,4 +51,4 @@ "gpu": "optional" }, "remoteUser": "devcontainer" -} \ No newline at end of file +} diff --git a/.devcontainer/ubuntu20/devcontainer.json b/.devcontainer/ubuntu20/devcontainer.json index b3321a836..e77831844 100644 --- a/.devcontainer/ubuntu20/devcontainer.json +++ b/.devcontainer/ubuntu20/devcontainer.json @@ -30,7 +30,8 @@ "matepek.vscode-catch2-test-adapter", "llvm-vs-code-extensions.vscode-clangd", "xaver.clang-format", - "bierner.markdown-mermaid" + "bierner.markdown-mermaid", + "rust-lang.rust-analyzer" ], "settings": { "C_Cpp.intelliSenseEngine": "disabled", @@ -52,4 +53,4 @@ "gpu": "optional" }, "remoteUser": "devcontainer" -} \ No newline at end of file +} diff --git a/.devcontainer/ubuntu22/devcontainer.json b/.devcontainer/ubuntu22/devcontainer.json index 57c7dbd34..02c77d4d7 100644 --- a/.devcontainer/ubuntu22/devcontainer.json +++ b/.devcontainer/ubuntu22/devcontainer.json @@ -30,7 +30,8 @@ "matepek.vscode-catch2-test-adapter", "llvm-vs-code-extensions.vscode-clangd", "xaver.clang-format", - "bierner.markdown-mermaid" + "bierner.markdown-mermaid", + "rust-lang.rust-analyzer" ], "settings": { "C_Cpp.intelliSenseEngine": "disabled", @@ -52,4 +53,4 @@ "gpu": "optional" }, "remoteUser": "devcontainer" -} \ No newline at end of file +} diff --git a/.devcontainer/ubuntu24/devcontainer.json b/.devcontainer/ubuntu24/devcontainer.json index 9777cf7f5..52f08331e 100644 --- a/.devcontainer/ubuntu24/devcontainer.json +++ b/.devcontainer/ubuntu24/devcontainer.json @@ -30,7 +30,8 @@ "matepek.vscode-catch2-test-adapter", "llvm-vs-code-extensions.vscode-clangd", "xaver.clang-format", - "bierner.markdown-mermaid" + "bierner.markdown-mermaid", + "rust-lang.rust-analyzer" ], "settings": { "C_Cpp.intelliSenseEngine": "disabled", @@ -52,4 +53,4 @@ "gpu": "optional" }, "remoteUser": "devcontainer" -} \ No newline at end of file +} diff --git a/.devcontainer/ubuntu25/devcontainer.json b/.devcontainer/ubuntu25/devcontainer.json index 1343790ec..87ea80b54 100644 --- a/.devcontainer/ubuntu25/devcontainer.json +++ b/.devcontainer/ubuntu25/devcontainer.json @@ -30,7 +30,8 @@ "matepek.vscode-catch2-test-adapter", "llvm-vs-code-extensions.vscode-clangd", "xaver.clang-format", - "bierner.markdown-mermaid" + "bierner.markdown-mermaid", + "rust-lang.rust-analyzer" ], "settings": { "C_Cpp.intelliSenseEngine": "disabled", @@ -52,4 +53,4 @@ "gpu": "optional" }, "remoteUser": "devcontainer" -} \ No newline at end of file +} From 483f6dff1a345f7bd7714e507a922b1e52f0c9d9 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 12 Aug 2025 10:56:22 +0100 Subject: [PATCH 53/77] reuse: add licensing information to Rust files Signed-off-by: Pedro Ferreira --- .github/workflows/rust.yml | 3 +++ rust/.gitattributes | 3 +++ rust/.gitignore | 3 +++ rust/Cargo.lock.license | 2 ++ rust/Cargo.toml | 3 +++ rust/README.md | 5 +++++ rust/flake.nix | 3 +++ rust/mxl-sys/.gitignore | 3 +++ rust/mxl-sys/Cargo.toml | 3 +++ rust/mxl-sys/build.rs | 3 +++ rust/mxl-sys/src/lib.rs | 3 +++ rust/mxl-sys/tests/simple_test.rs | 3 +++ rust/mxl-sys/wrapper-with-version-h.h | 3 +++ rust/mxl-sys/wrapper-without-version-h.h | 3 +++ rust/mxl/Cargo.toml | 3 +++ rust/mxl/build.rs | 3 +++ rust/mxl/examples/common/mod.rs | 3 +++ rust/mxl/examples/flow-reader.rs | 3 +++ rust/mxl/examples/flow-writer.rs | 3 +++ rust/mxl/src/api.rs | 3 +++ rust/mxl/src/config.rs | 3 +++ rust/mxl/src/error.rs | 3 +++ rust/mxl/src/flow.rs | 3 +++ rust/mxl/src/flow/reader.rs | 3 +++ rust/mxl/src/flow/writer.rs | 3 +++ rust/mxl/src/grain.rs | 3 +++ rust/mxl/src/grain/data.rs | 3 +++ rust/mxl/src/grain/reader.rs | 3 +++ rust/mxl/src/grain/write_access.rs | 3 +++ rust/mxl/src/grain/writer.rs | 3 +++ rust/mxl/src/instance.rs | 3 +++ rust/mxl/src/lib.rs | 3 +++ rust/mxl/src/samples.rs | 3 +++ rust/mxl/src/samples/data.rs | 3 +++ rust/mxl/src/samples/reader.rs | 3 +++ rust/mxl/src/samples/write_access.rs | 3 +++ rust/mxl/src/samples/writer.rs | 3 +++ rust/mxl/src/tools.rs | 3 +++ rust/mxl/tests/basic_tests.rs | 3 +++ 39 files changed, 118 insertions(+) create mode 100644 rust/Cargo.lock.license diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bbacba38c..a2fa3889b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 + name: Test the rust bindings on: diff --git a/rust/.gitattributes b/rust/.gitattributes index 383338e04..265e6dd73 100644 --- a/rust/.gitattributes +++ b/rust/.gitattributes @@ -1 +1,4 @@ +# SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 + Cargo.lock -diff diff --git a/rust/.gitignore b/rust/.gitignore index 09c22a059..91712988d 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 + .DS_Store .idea .vscode diff --git a/rust/Cargo.lock.license b/rust/Cargo.lock.license new file mode 100644 index 000000000..e8d14bb3c --- /dev/null +++ b/rust/Cargo.lock.license @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 65ca6f082..d986b3aa9 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 + [workspace] members = ["mxl", "mxl-sys"] diff --git a/rust/README.md b/rust/README.md index 8dcb8a225..d2e6bc81a 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1,3 +1,8 @@ + + # Rust bindings for DMF MXL ## Goals diff --git a/rust/flake.nix b/rust/flake.nix index a219b532b..eab9efe98 100644 --- a/rust/flake.nix +++ b/rust/flake.nix @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 + { description = "Flake for MXL dev"; diff --git a/rust/mxl-sys/.gitignore b/rust/mxl-sys/.gitignore index 8d6e34004..bbf8a55df 100644 --- a/rust/mxl-sys/.gitignore +++ b/rust/mxl-sys/.gitignore @@ -1 +1,4 @@ +# SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 + mxl/version.h diff --git a/rust/mxl-sys/Cargo.toml b/rust/mxl-sys/Cargo.toml index 0e4fa65a9..58e1dbf19 100644 --- a/rust/mxl-sys/Cargo.toml +++ b/rust/mxl-sys/Cargo.toml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 + [package] name = "mxl-sys" edition.workspace = true diff --git a/rust/mxl-sys/build.rs b/rust/mxl-sys/build.rs index 2eaf1d9f3..4154ed030 100644 --- a/rust/mxl-sys/build.rs +++ b/rust/mxl-sys/build.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::env; use std::path::PathBuf; diff --git a/rust/mxl-sys/src/lib.rs b/rust/mxl-sys/src/lib.rs index 2e4dc43dd..d82df1125 100644 --- a/rust/mxl-sys/src/lib.rs +++ b/rust/mxl-sys/src/lib.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + #![allow(non_upper_case_globals)] #![allow(non_camel_case_types)] #![allow(non_snake_case)] diff --git a/rust/mxl-sys/tests/simple_test.rs b/rust/mxl-sys/tests/simple_test.rs index c617fe566..d0357dc46 100644 --- a/rust/mxl-sys/tests/simple_test.rs +++ b/rust/mxl-sys/tests/simple_test.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + #[test] fn there_is_bindgen_generated_code() { let mxl_version = mxl_sys::mxlVersionType { diff --git a/rust/mxl-sys/wrapper-with-version-h.h b/rust/mxl-sys/wrapper-with-version-h.h index 80c5254f5..44b4bd777 100644 --- a/rust/mxl-sys/wrapper-with-version-h.h +++ b/rust/mxl-sys/wrapper-with-version-h.h @@ -1,2 +1,5 @@ +// SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + #include "wrapper-without-version-h.h" #include "mxl/version.h" diff --git a/rust/mxl-sys/wrapper-without-version-h.h b/rust/mxl-sys/wrapper-without-version-h.h index 5248f1e1f..e6fb42167 100644 --- a/rust/mxl-sys/wrapper-without-version-h.h +++ b/rust/mxl-sys/wrapper-without-version-h.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + #include "mxl/dataformat.h" #include "mxl/flow.h" #include "mxl/flowinfo.h" diff --git a/rust/mxl/Cargo.toml b/rust/mxl/Cargo.toml index 447c49987..325c59ecf 100644 --- a/rust/mxl/Cargo.toml +++ b/rust/mxl/Cargo.toml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 + [package] name = "mxl" edition.workspace = true diff --git a/rust/mxl/build.rs b/rust/mxl/build.rs index 382d75e18..4451cc68a 100644 --- a/rust/mxl/build.rs +++ b/rust/mxl/build.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::env; use std::path::PathBuf; diff --git a/rust/mxl/examples/common/mod.rs b/rust/mxl/examples/common/mod.rs index 221a46291..e37eb881b 100644 --- a/rust/mxl/examples/common/mod.rs +++ b/rust/mxl/examples/common/mod.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + pub fn setup_logging() { tracing_subscriber::fmt() .with_env_filter( diff --git a/rust/mxl/examples/flow-reader.rs b/rust/mxl/examples/flow-reader.rs index 5fce97946..60987c878 100644 --- a/rust/mxl/examples/flow-reader.rs +++ b/rust/mxl/examples/flow-reader.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + mod common; use std::time::Duration; diff --git a/rust/mxl/examples/flow-writer.rs b/rust/mxl/examples/flow-writer.rs index b91bfc3a8..67ca8b095 100644 --- a/rust/mxl/examples/flow-writer.rs +++ b/rust/mxl/examples/flow-writer.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + mod common; use clap::Parser; diff --git a/rust/mxl/src/api.rs b/rust/mxl/src/api.rs index 5cf81e9f0..2c9ab3f9e 100644 --- a/rust/mxl/src/api.rs +++ b/rust/mxl/src/api.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::path::Path; use dlopen2::wrapper::{Container, WrapperApi}; diff --git a/rust/mxl/src/config.rs b/rust/mxl/src/config.rs index d40018708..69295d477 100644 --- a/rust/mxl/src/config.rs +++ b/rust/mxl/src/config.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::str::FromStr; include!(concat!(env!("OUT_DIR"), "/constants.rs")); diff --git a/rust/mxl/src/error.rs b/rust/mxl/src/error.rs index a56a4aad6..d5372febf 100644 --- a/rust/mxl/src/error.rs +++ b/rust/mxl/src/error.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + pub type Result = core::result::Result; #[derive(Debug, thiserror::Error)] diff --git a/rust/mxl/src/flow.rs b/rust/mxl/src/flow.rs index d82e0eca1..d0b3f7531 100644 --- a/rust/mxl/src/flow.rs +++ b/rust/mxl/src/flow.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + pub mod reader; pub mod writer; diff --git a/rust/mxl/src/flow/reader.rs b/rust/mxl/src/flow/reader.rs index a5bfd6787..82aa123a9 100644 --- a/rust/mxl/src/flow/reader.rs +++ b/rust/mxl/src/flow/reader.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::sync::Arc; use crate::{ diff --git a/rust/mxl/src/flow/writer.rs b/rust/mxl/src/flow/writer.rs index 250578db0..b1234073d 100644 --- a/rust/mxl/src/flow/writer.rs +++ b/rust/mxl/src/flow/writer.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::sync::Arc; use crate::{ diff --git a/rust/mxl/src/grain.rs b/rust/mxl/src/grain.rs index 9bfae8f03..2205933c9 100644 --- a/rust/mxl/src/grain.rs +++ b/rust/mxl/src/grain.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + pub mod data; pub mod reader; pub mod write_access; diff --git a/rust/mxl/src/grain/data.rs b/rust/mxl/src/grain/data.rs index fabdd1354..d8cd98797 100644 --- a/rust/mxl/src/grain/data.rs +++ b/rust/mxl/src/grain/data.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + pub struct GrainData<'a> { pub user_data: &'a [u8], pub payload: &'a [u8], diff --git a/rust/mxl/src/grain/reader.rs b/rust/mxl/src/grain/reader.rs index 227e3e8ab..8ecdec7fe 100644 --- a/rust/mxl/src/grain/reader.rs +++ b/rust/mxl/src/grain/reader.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::{sync::Arc, time::Duration}; use crate::{ diff --git a/rust/mxl/src/grain/write_access.rs b/rust/mxl/src/grain/write_access.rs index 04447aef4..69657eb99 100644 --- a/rust/mxl/src/grain/write_access.rs +++ b/rust/mxl/src/grain/write_access.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::{marker::PhantomData, sync::Arc}; use tracing::error; diff --git a/rust/mxl/src/grain/writer.rs b/rust/mxl/src/grain/writer.rs index a2399e0b3..2c06b7089 100644 --- a/rust/mxl/src/grain/writer.rs +++ b/rust/mxl/src/grain/writer.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::sync::Arc; use super::write_access::GrainWriteAccess; diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index 95861b5ec..8e703cd18 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::{ffi::CString, sync::Arc}; use dlopen2::wrapper::Container; diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index b2ce1e5e5..723739d70 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + mod api; mod error; mod flow; diff --git a/rust/mxl/src/samples.rs b/rust/mxl/src/samples.rs index 9bfae8f03..2205933c9 100644 --- a/rust/mxl/src/samples.rs +++ b/rust/mxl/src/samples.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + pub mod data; pub mod reader; pub mod write_access; diff --git a/rust/mxl/src/samples/data.rs b/rust/mxl/src/samples/data.rs index 4c0796908..ef7797d97 100644 --- a/rust/mxl/src/samples/data.rs +++ b/rust/mxl/src/samples/data.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::marker::PhantomData; use crate::Error; diff --git a/rust/mxl/src/samples/reader.rs b/rust/mxl/src/samples/reader.rs index e6f75dcdd..5612f3191 100644 --- a/rust/mxl/src/samples/reader.rs +++ b/rust/mxl/src/samples/reader.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::sync::Arc; use crate::{ diff --git a/rust/mxl/src/samples/write_access.rs b/rust/mxl/src/samples/write_access.rs index bbb57ce1a..08fe96326 100644 --- a/rust/mxl/src/samples/write_access.rs +++ b/rust/mxl/src/samples/write_access.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::{marker::PhantomData, sync::Arc}; use tracing::error; diff --git a/rust/mxl/src/samples/writer.rs b/rust/mxl/src/samples/writer.rs index 4c0193f50..f1e9ac370 100644 --- a/rust/mxl/src/samples/writer.rs +++ b/rust/mxl/src/samples/writer.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + use std::sync::Arc; use crate::{Error, Result, SamplesWriteAccess, instance::InstanceContext}; diff --git a/rust/mxl/src/tools.rs b/rust/mxl/src/tools.rs index 312af2180..156e6593c 100644 --- a/rust/mxl/src/tools.rs +++ b/rust/mxl/src/tools.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + pub fn read_file(file_path: impl AsRef) -> Result { use std::io::Read; diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index fb5295bf0..544f8c0d2 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +// SPDX-License-Identifier: Apache-2.0 + /// Tests of the basic low level synchronous API. /// /// The tests now require an MXL library of a specific name to be present in the system. This should From cf998ad45df03c143c1aa0a7bd678d4adcb0bc0d Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 12 Aug 2025 16:57:38 +0100 Subject: [PATCH 54/77] rust test: add a nextest config file Signed-off-by: Pedro Ferreira --- rust/.config/nextest.toml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 rust/.config/nextest.toml diff --git a/rust/.config/nextest.toml b/rust/.config/nextest.toml new file mode 100644 index 000000000..6a109e171 --- /dev/null +++ b/rust/.config/nextest.toml @@ -0,0 +1,5 @@ +[profile.ci] +fail-fast = false + +[profile.ci.junit] +path = "junit.xml" From f78f26cd151cce8ab49715635a5b3fc81519dca0 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Mon, 25 Aug 2025 16:46:08 +0200 Subject: [PATCH 55/77] Make Rust bindings compatiple with MXL 0.7.x Signed-off-by: Pavel Cernohorsky --- rust/mxl/src/api.rs | 24 ++++++++++++------------ rust/mxl/src/flow.rs | 8 ++++---- rust/mxl/src/flow/reader.rs | 2 +- rust/mxl/src/grain/reader.rs | 2 +- rust/mxl/src/grain/write_access.rs | 4 ++-- rust/mxl/src/grain/writer.rs | 2 +- rust/mxl/src/instance.rs | 10 +++++----- rust/mxl/src/samples/data.rs | 4 ++-- rust/mxl/src/samples/reader.rs | 2 +- rust/mxl/src/samples/write_access.rs | 4 ++-- rust/mxl/src/samples/writer.rs | 2 +- 11 files changed, 32 insertions(+), 32 deletions(-) diff --git a/rust/mxl/src/api.rs b/rust/mxl/src/api.rs index 2c9ab3f9e..19e27fcc6 100644 --- a/rust/mxl/src/api.rs +++ b/rust/mxl/src/api.rs @@ -30,7 +30,7 @@ pub struct MxlApi { instance: mxl_sys::mxlInstance, flowDef: *const std::os::raw::c_char, options: *const std::os::raw::c_char, - info: *mut mxl_sys::FlowInfo, + info: *mut mxl_sys::mxlFlowInfo, ) -> mxl_sys::mxlStatus, #[allow(non_snake_case)] #[dlopen2_name = "mxlDestroyFlow"] @@ -67,7 +67,7 @@ pub struct MxlApi { #[dlopen2_name = "mxlFlowReaderGetInfo"] mxl_flow_reader_get_info: unsafe extern "C" fn( reader: mxl_sys::mxlFlowReader, - info: *mut mxl_sys::FlowInfo, + info: *mut mxl_sys::mxlFlowInfo, ) -> mxl_sys::mxlStatus, #[allow(non_snake_case)] #[dlopen2_name = "mxlFlowReaderGetGrain"] @@ -75,14 +75,14 @@ pub struct MxlApi { reader: mxl_sys::mxlFlowReader, index: u64, timeoutNs: u64, - grain: *mut mxl_sys::GrainInfo, + grain: *mut mxl_sys::mxlGrainInfo, payload: *mut *mut u8, ) -> mxl_sys::mxlStatus, #[dlopen2_name = "mxlFlowReaderGetGrainNonBlocking"] mxl_flow_reader_get_grain_non_blocking: unsafe extern "C" fn( reader: mxl_sys::mxlFlowReader, index: u64, - grain: *mut mxl_sys::GrainInfo, + grain: *mut mxl_sys::mxlGrainInfo, payload: *mut *mut u8, ) -> mxl_sys::mxlStatus, #[allow(non_snake_case)] @@ -90,7 +90,7 @@ pub struct MxlApi { mxl_flow_writer_open_grain: unsafe extern "C" fn( writer: mxl_sys::mxlFlowWriter, index: u64, - grainInfo: *mut mxl_sys::GrainInfo, + grainInfo: *mut mxl_sys::mxlGrainInfo, payload: *mut *mut u8, ) -> mxl_sys::mxlStatus, #[dlopen2_name = "mxlFlowWriterCancelGrain"] @@ -99,7 +99,7 @@ pub struct MxlApi { #[dlopen2_name = "mxlFlowWriterCommitGrain"] mxl_flow_writer_commit_grain: unsafe extern "C" fn( writer: mxl_sys::mxlFlowWriter, - grain: *const mxl_sys::GrainInfo, + grain: *const mxl_sys::mxlGrainInfo, ) -> mxl_sys::mxlStatus, #[allow(non_snake_case)] #[dlopen2_name = "mxlFlowReaderGetSamples"] @@ -107,7 +107,7 @@ pub struct MxlApi { reader: mxl_sys::mxlFlowReader, index: u64, count: usize, - payloadBuffersSlices: *mut mxl_sys::WrappedMultiBufferSlice, + payloadBuffersSlices: *mut mxl_sys::mxlWrappedMultiBufferSlice, ) -> mxl_sys::mxlStatus, #[allow(non_snake_case)] #[dlopen2_name = "mxlFlowWriterOpenSamples"] @@ -115,7 +115,7 @@ pub struct MxlApi { writer: mxl_sys::mxlFlowWriter, index: u64, count: usize, - payloadBuffersSlices: *mut mxl_sys::MutableWrappedMultiBufferSlice, + payloadBuffersSlices: *mut mxl_sys::mxlMutableWrappedMultiBufferSlice, ) -> mxl_sys::mxlStatus, #[dlopen2_name = "mxlFlowWriterCancelSamples"] mxl_flow_writer_cancel_samples: @@ -125,19 +125,19 @@ pub struct MxlApi { unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, #[allow(non_snake_case)] #[dlopen2_name = "mxlGetCurrentIndex"] - mxl_get_current_index: unsafe extern "C" fn(editRate: *const mxl_sys::Rational) -> u64, + mxl_get_current_index: unsafe extern "C" fn(editRate: *const mxl_sys::mxlRational) -> u64, #[allow(non_snake_case)] #[dlopen2_name = "mxlGetNsUntilIndex"] mxl_get_ns_until_index: - unsafe extern "C" fn(index: u64, editRate: *const mxl_sys::Rational) -> u64, + unsafe extern "C" fn(index: u64, editRate: *const mxl_sys::mxlRational) -> u64, #[allow(non_snake_case)] #[dlopen2_name = "mxlTimestampToIndex"] mxl_timestamp_to_index: - unsafe extern "C" fn(editRate: *const mxl_sys::Rational, timestamp: u64) -> u64, + unsafe extern "C" fn(editRate: *const mxl_sys::mxlRational, timestamp: u64) -> u64, #[allow(non_snake_case)] #[dlopen2_name = "mxlIndexToTimestamp"] mxl_index_to_timestamp: - unsafe extern "C" fn(editRate: *const mxl_sys::Rational, index: u64) -> u64, + unsafe extern "C" fn(editRate: *const mxl_sys::mxlRational, index: u64) -> u64, #[dlopen2_name = "mxlSleepForNs"] mxl_sleep_for_ns: unsafe extern "C" fn(ns: u64), #[dlopen2_name = "mxlGetTime"] diff --git a/rust/mxl/src/flow.rs b/rust/mxl/src/flow.rs index d0b3f7531..b91cd4f9c 100644 --- a/rust/mxl/src/flow.rs +++ b/rust/mxl/src/flow.rs @@ -36,11 +36,11 @@ pub(crate) fn is_discrete_data_format(format: u32) -> bool { } pub struct FlowInfo { - pub(crate) value: mxl_sys::FlowInfo, + pub(crate) value: mxl_sys::mxlFlowInfo, } impl FlowInfo { - pub fn discrete_flow_info(&self) -> Result<&mxl_sys::DiscreteFlowInfo> { + pub fn discrete_flow_info(&self) -> Result<&mxl_sys::mxlDiscreteFlowInfo> { if !is_discrete_data_format(self.value.common.format) { return Err(Error::Other(format!( "Flow format is {}, video or data required.", @@ -50,7 +50,7 @@ impl FlowInfo { Ok(unsafe { &self.value.__bindgen_anon_1.discrete }) } - pub fn continuous_flow_info(&self) -> Result<&mxl_sys::ContinuousFlowInfo> { + pub fn continuous_flow_info(&self) -> Result<&mxl_sys::mxlContinuousFlowInfo> { if is_discrete_data_format(self.value.common.format) { return Err(Error::Other(format!( "Flow format is {}, audio required.", @@ -69,7 +69,7 @@ impl FlowInfo { } } -pub struct CommonFlowInfo<'a>(&'a mxl_sys::CommonFlowInfo); +pub struct CommonFlowInfo<'a>(&'a mxl_sys::mxlCommonFlowInfo); impl CommonFlowInfo<'_> { pub fn id(&self) -> Uuid { diff --git a/rust/mxl/src/flow/reader.rs b/rust/mxl/src/flow/reader.rs index 82aa123a9..38abfd73c 100644 --- a/rust/mxl/src/flow/reader.rs +++ b/rust/mxl/src/flow/reader.rs @@ -22,7 +22,7 @@ pub(crate) fn get_flow_info( context: &Arc, reader: mxl_sys::mxlFlowReader, ) -> Result { - let mut flow_info: mxl_sys::FlowInfo = unsafe { std::mem::zeroed() }; + let mut flow_info: mxl_sys::mxlFlowInfo = unsafe { std::mem::zeroed() }; unsafe { Error::from_status(context.api.mxl_flow_reader_get_info(reader, &mut flow_info))?; } diff --git a/rust/mxl/src/grain/reader.rs b/rust/mxl/src/grain/reader.rs index 8ecdec7fe..df892e025 100644 --- a/rust/mxl/src/grain/reader.rs +++ b/rust/mxl/src/grain/reader.rs @@ -36,7 +36,7 @@ impl GrainReader { index: u64, timeout: Duration, ) -> Result> { - let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; + let mut grain_info: mxl_sys::mxlGrainInfo = unsafe { std::mem::zeroed() }; let mut payload_ptr: *mut u8 = std::ptr::null_mut(); let timeout_ns = timeout.as_nanos() as u64; loop { diff --git a/rust/mxl/src/grain/write_access.rs b/rust/mxl/src/grain/write_access.rs index 69657eb99..c34696086 100644 --- a/rust/mxl/src/grain/write_access.rs +++ b/rust/mxl/src/grain/write_access.rs @@ -13,7 +13,7 @@ use crate::{Error, Result, instance::InstanceContext}; pub struct GrainWriteAccess<'a> { context: Arc, writer: mxl_sys::mxlFlowWriter, - grain_info: mxl_sys::GrainInfo, + grain_info: mxl_sys::mxlGrainInfo, payload_ptr: *mut u8, /// Serves as a flag to know whether to cancel the grain on drop. committed_or_canceled: bool, @@ -24,7 +24,7 @@ impl<'a> GrainWriteAccess<'a> { pub(crate) fn new( context: Arc, writer: mxl_sys::mxlFlowWriter, - grain_info: mxl_sys::GrainInfo, + grain_info: mxl_sys::mxlGrainInfo, payload_ptr: *mut u8, ) -> Self { Self { diff --git a/rust/mxl/src/grain/writer.rs b/rust/mxl/src/grain/writer.rs index 2c06b7089..85e2598d6 100644 --- a/rust/mxl/src/grain/writer.rs +++ b/rust/mxl/src/grain/writer.rs @@ -31,7 +31,7 @@ impl GrainWriter { /// multiple grains. If the TODO ever gets removed, it may be worth considering pattern where /// opening grain would consume the writer and then return it back on commit or cancel. pub fn open_grain<'a>(&'a self, index: u64) -> Result> { - let mut grain_info: mxl_sys::GrainInfo = unsafe { std::mem::zeroed() }; + let mut grain_info: mxl_sys::mxlGrainInfo = unsafe { std::mem::zeroed() }; let mut payload_ptr: *mut u8 = std::ptr::null_mut(); unsafe { Error::from_status(self.context.api.mxl_flow_writer_open_grain( diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index 8e703cd18..878eb7c4f 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -115,7 +115,7 @@ impl MxlInstance { pub fn create_flow(&self, flow_def: &str, options: Option<&str>) -> Result { let flow_def = CString::new(flow_def)?; let options = CString::new(options.unwrap_or(""))?; - let mut info = std::mem::MaybeUninit::::uninit(); + let mut info = std::mem::MaybeUninit::::uninit(); unsafe { Error::from_status(self.context.api.mxl_create_flow( @@ -143,14 +143,14 @@ impl MxlInstance { Ok(()) } - pub fn get_current_index(&self, rational: &mxl_sys::Rational) -> u64 { + pub fn get_current_index(&self, rational: &mxl_sys::mxlRational) -> u64 { unsafe { self.context.api.mxl_get_current_index(rational) } } pub fn get_duration_until_index( &self, index: u64, - rate: &mxl_sys::Rational, + rate: &mxl_sys::mxlRational, ) -> Result { let duration_ns = unsafe { self.context.api.mxl_get_ns_until_index(index, rate) }; if duration_ns == u64::MAX { @@ -164,7 +164,7 @@ impl MxlInstance { } /// TODO: Make timestamp a strong type. - pub fn timestamp_to_index(&self, timestamp: u64, rate: &mxl_sys::Rational) -> Result { + pub fn timestamp_to_index(&self, timestamp: u64, rate: &mxl_sys::mxlRational) -> Result { let index = unsafe { self.context.api.mxl_timestamp_to_index(rate, timestamp) }; if index == u64::MAX { Err(Error::Other(format!( @@ -176,7 +176,7 @@ impl MxlInstance { } } - pub fn index_to_timestamp(&self, index: u64, rate: &mxl_sys::Rational) -> Result { + pub fn index_to_timestamp(&self, index: u64, rate: &mxl_sys::mxlRational) -> Result { let timestamp = unsafe { self.context.api.mxl_index_to_timestamp(rate, index) }; if timestamp == u64::MAX { Err(Error::Other(format!( diff --git a/rust/mxl/src/samples/data.rs b/rust/mxl/src/samples/data.rs index ef7797d97..08e176035 100644 --- a/rust/mxl/src/samples/data.rs +++ b/rust/mxl/src/samples/data.rs @@ -6,12 +6,12 @@ use std::marker::PhantomData; use crate::Error; pub struct SamplesData<'a> { - buffer_slice: mxl_sys::WrappedMultiBufferSlice, + buffer_slice: mxl_sys::mxlWrappedMultiBufferSlice, phantom: PhantomData<&'a ()>, } impl<'a> SamplesData<'a> { - pub(crate) fn new(buffer_slice: mxl_sys::WrappedMultiBufferSlice) -> Self { + pub(crate) fn new(buffer_slice: mxl_sys::mxlWrappedMultiBufferSlice) -> Self { Self { buffer_slice, phantom: Default::default(), diff --git a/rust/mxl/src/samples/reader.rs b/rust/mxl/src/samples/reader.rs index 5612f3191..9c31d7e6a 100644 --- a/rust/mxl/src/samples/reader.rs +++ b/rust/mxl/src/samples/reader.rs @@ -32,7 +32,7 @@ impl SamplesReader { } pub fn get_samples(&self, index: u64, count: usize) -> Result> { - let mut buffer_slice: mxl_sys::WrappedMultiBufferSlice = unsafe { std::mem::zeroed() }; + let mut buffer_slice: mxl_sys::mxlWrappedMultiBufferSlice = unsafe { std::mem::zeroed() }; unsafe { Error::from_status(self.context.api.mxl_flow_reader_get_samples( self.reader, diff --git a/rust/mxl/src/samples/write_access.rs b/rust/mxl/src/samples/write_access.rs index 08fe96326..5b43c53ea 100644 --- a/rust/mxl/src/samples/write_access.rs +++ b/rust/mxl/src/samples/write_access.rs @@ -16,7 +16,7 @@ use crate::{Error, instance::InstanceContext}; pub struct SamplesWriteAccess<'a> { context: Arc, writer: mxl_sys::mxlFlowWriter, - buffer_slice: mxl_sys::MutableWrappedMultiBufferSlice, + buffer_slice: mxl_sys::mxlMutableWrappedMultiBufferSlice, /// Serves as a flag to know whether to cancel the samples on drop. committed_or_canceled: bool, phantom: PhantomData<&'a ()>, @@ -26,7 +26,7 @@ impl<'a> SamplesWriteAccess<'a> { pub(crate) fn new( context: Arc, writer: mxl_sys::mxlFlowWriter, - buffer_slice: mxl_sys::MutableWrappedMultiBufferSlice, + buffer_slice: mxl_sys::mxlMutableWrappedMultiBufferSlice, ) -> Self { Self { context, diff --git a/rust/mxl/src/samples/writer.rs b/rust/mxl/src/samples/writer.rs index f1e9ac370..ecc455b47 100644 --- a/rust/mxl/src/samples/writer.rs +++ b/rust/mxl/src/samples/writer.rs @@ -25,7 +25,7 @@ impl SamplesWriter { } pub fn open_samples<'a>(&'a self, index: u64, count: usize) -> Result> { - let mut buffer_slice: mxl_sys::MutableWrappedMultiBufferSlice = + let mut buffer_slice: mxl_sys::mxlMutableWrappedMultiBufferSlice = unsafe { std::mem::zeroed() }; unsafe { Error::from_status(self.context.api.mxl_flow_writer_open_samples( From 1d1b021622dfb8c2af623115e8a9f615fef4e451 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Thu, 28 Aug 2025 08:09:01 +0200 Subject: [PATCH 56/77] Add mxlGetFlowDef into the Rust bindings Signed-off-by: Pavel Cernohorsky --- rust/mxl/src/api.rs | 10 +++++++++ rust/mxl/src/instance.rs | 38 +++++++++++++++++++++++++++++++++++ rust/mxl/tests/basic_tests.rs | 25 +++++++++++++++++++---- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/rust/mxl/src/api.rs b/rust/mxl/src/api.rs index 19e27fcc6..c47c18d03 100644 --- a/rust/mxl/src/api.rs +++ b/rust/mxl/src/api.rs @@ -38,6 +38,16 @@ pub struct MxlApi { instance: mxl_sys::mxlInstance, flowId: *const std::os::raw::c_char, ) -> mxl_sys::mxlStatus, + + #[allow(non_snake_case)] + #[dlopen2_name = "mxlGetFlowDef"] + mxl_get_flow_def: unsafe extern "C" fn( + instance: mxl_sys::mxlInstance, + flowId: *const ::std::os::raw::c_char, + buffer: *mut ::std::os::raw::c_char, + bufferSize: *mut usize, + ) -> mxl_sys::mxlStatus, + #[allow(non_snake_case)] #[dlopen2_name = "mxlCreateFlowReader"] mxl_create_flow_reader: unsafe extern "C" fn( diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index 878eb7c4f..64fd5d2a3 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -143,6 +143,44 @@ impl MxlInstance { Ok(()) } + pub fn get_flow_def(&self, flow_id: &str) -> Result { + let flow_id = CString::new(flow_id)?; + const INITIAL_BUFFER_SIZE: usize = 4096; + let mut buffer: Vec = vec![0; INITIAL_BUFFER_SIZE]; + let mut buffer_size = INITIAL_BUFFER_SIZE; + + let status = unsafe { + self.context.api.mxl_get_flow_def( + self.context.instance, + flow_id.as_ptr(), + buffer.as_mut_ptr() as *mut std::os::raw::c_char, + &mut buffer_size, + ) + }; + + if status == mxl_sys::MXL_ERR_INVALID_ARG && buffer_size > INITIAL_BUFFER_SIZE { + buffer = vec![0; buffer_size]; + unsafe { + Error::from_status(self.context.api.mxl_get_flow_def( + self.context.instance, + flow_id.as_ptr(), + buffer.as_mut_ptr() as *mut std::os::raw::c_char, + &mut buffer_size, + ))?; + } + } else { + Error::from_status(status)?; + } + + if buffer_size > 0 && buffer[buffer_size - 1] == 0 { + buffer_size -= 1; + } + buffer.truncate(buffer_size); + + String::from_utf8(buffer) + .map_err(|_| Error::Other("Invalid UTF-8 in flow definition".to_string())) + } + pub fn get_current_index(&self, rational: &mxl_sys::mxlRational) -> u64 { unsafe { self.context.api.mxl_get_current_index(rational) } } diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index 544f8c0d2..9c37ae1a3 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -40,10 +40,7 @@ fn setup_test(test: &str) -> mxl::MxlInstance { mxl::MxlInstance::new(mxl_api, &domain, "").unwrap() } -fn prepare_flow_info>( - mxl_instance: &MxlInstance, - path: P, -) -> mxl::FlowInfo { +fn read_flow_def>(path: P) -> String { let flow_config_file = mxl::config::get_mxl_repo_root().join(path); let flow_def = mxl::tools::read_file(flow_config_file.as_path()) .map_err(|error| { @@ -54,6 +51,14 @@ fn prepare_flow_info>( )) }) .unwrap(); + flow_def +} + +fn prepare_flow_info>( + mxl_instance: &MxlInstance, + path: P, +) -> mxl::FlowInfo { + let flow_def = read_flow_def(path); mxl_instance.create_flow(flow_def.as_str(), None).unwrap() } @@ -107,3 +112,15 @@ fn basic_mxl_samples_writing_reading() { mxl_instance.destroy_flow(flow_id.as_str()).unwrap(); mxl_instance.destroy().unwrap(); } + +#[test] +fn get_flow_def() { + let mxl_instance = setup_test("flow_def"); + let flow_def = read_flow_def("lib/tests/data/v210_flow.json"); + let flow_info = mxl_instance.create_flow(flow_def.as_str(), None).unwrap(); + let flow_id = flow_info.common_flow_info().id().to_string(); + let retrieved_flow_def = mxl_instance.get_flow_def(flow_id.as_str()).unwrap(); + assert_eq!(flow_def, retrieved_flow_def); + mxl_instance.destroy_flow(flow_id.as_str()).unwrap(); + mxl_instance.destroy().unwrap(); +} From fde4dc6103b1f5b008f7f10895f4c672d8bf77c4 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Fri, 26 Sep 2025 14:38:11 +0200 Subject: [PATCH 57/77] Fix mxl-not-built feature This feature makes the MXL Rust bindings build skip building the MXL library itself and uses the one in the main build directory of MXL instead. It got broken after recent changes. Signed-off-by: Pavel Cernohorsky --- rust/mxl/src/config.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/rust/mxl/src/config.rs b/rust/mxl/src/config.rs index 69295d477..b80fd0c2b 100644 --- a/rust/mxl/src/config.rs +++ b/rust/mxl/src/config.rs @@ -5,12 +5,21 @@ use std::str::FromStr; include!(concat!(env!("OUT_DIR"), "/constants.rs")); +#[cfg(not(feature = "mxl-not-built"))] pub fn get_mxl_so_path() -> std::path::PathBuf { // The mxl-sys build script ensures that the build directory is in the library path // so we can just return the library name here. "libmxl.so".into() } +#[cfg(feature = "mxl-not-built")] +pub fn get_mxl_so_path() -> std::path::PathBuf { + std::path::PathBuf::from_str(MXL_BUILD_DIR) + .expect("build error: 'MXL_BUILD_DIR' is invalid") + .join("lib") + .join("libmxl.so") +} + pub fn get_mxl_repo_root() -> std::path::PathBuf { std::path::PathBuf::from_str(MXL_REPO_ROOT).expect("build error: 'MXL_REPO_ROOT' is invalid") } From c82d779e8d598e0e293e6ef9a8deaed158929671 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Fri, 26 Sep 2025 14:47:51 +0200 Subject: [PATCH 58/77] Use std::fs::read_to_string instead of custom function Signed-off-by: Pavel Cernohorsky --- rust/mxl/examples/flow-writer.rs | 2 +- rust/mxl/src/lib.rs | 1 - rust/mxl/src/tools.rs | 11 ----------- rust/mxl/tests/basic_tests.rs | 2 +- 4 files changed, 2 insertions(+), 14 deletions(-) delete mode 100644 rust/mxl/src/tools.rs diff --git a/rust/mxl/examples/flow-writer.rs b/rust/mxl/examples/flow-writer.rs index 67ca8b095..7f369c80b 100644 --- a/rust/mxl/examples/flow-writer.rs +++ b/rust/mxl/examples/flow-writer.rs @@ -35,7 +35,7 @@ fn main() -> Result<(), mxl::Error> { let mxl_api = mxl::load_api(get_mxl_so_path())?; let mxl_instance = mxl::MxlInstance::new(mxl_api, &opts.mxl_domain, "")?; - let flow_def = mxl::tools::read_file(opts.flow_config_file.as_str()).map_err(|error| { + let flow_def = std::fs::read_to_string(opts.flow_config_file.as_str()).map_err(|error| { mxl::Error::Other(format!( "Error while reading flow definition from \"{}\": {}", &opts.flow_config_file, error diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index 723739d70..1b9ecbc9b 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -9,7 +9,6 @@ mod instance; mod samples; pub mod config; -pub mod tools; pub use api::{MxlApi, load_api}; pub use error::{Error, Result}; diff --git a/rust/mxl/src/tools.rs b/rust/mxl/src/tools.rs deleted file mode 100644 index 156e6593c..000000000 --- a/rust/mxl/src/tools.rs +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. -// SPDX-License-Identifier: Apache-2.0 - -pub fn read_file(file_path: impl AsRef) -> Result { - use std::io::Read; - - let mut file = std::fs::File::open(file_path)?; - let mut contents = String::new(); - file.read_to_string(&mut contents)?; - Ok(contents) -} diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index 9c37ae1a3..003952a32 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -42,7 +42,7 @@ fn setup_test(test: &str) -> mxl::MxlInstance { fn read_flow_def>(path: P) -> String { let flow_config_file = mxl::config::get_mxl_repo_root().join(path); - let flow_def = mxl::tools::read_file(flow_config_file.as_path()) + let flow_def = std::fs::read_to_string(flow_config_file.as_path()) .map_err(|error| { mxl::Error::Other(format!( "Error while reading flow definition from \"{}\": {}", From 72838b768d32ee0a3a0ed3daeaf4a3693342edbe Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 11:10:17 +0000 Subject: [PATCH 59/77] dev container: remove duplicate entry cargo audit was being installed twice Signed-off-by: Pedro Ferreira --- .devcontainer/scripts/common/rust/install-rust.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.devcontainer/scripts/common/rust/install-rust.sh b/.devcontainer/scripts/common/rust/install-rust.sh index fbab87790..253388cd5 100755 --- a/.devcontainer/scripts/common/rust/install-rust.sh +++ b/.devcontainer/scripts/common/rust/install-rust.sh @@ -25,5 +25,4 @@ cargo binstall cargo-outdated --locked cargo binstall cargo-machete --locked cargo binstall cargo-deny --locked -cargo binstall cargo-audit --locked cargo binstall cargo-nextest --locked From 218845bfd5e5b40351f97604c0ff53a60afa79ef Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 16:15:19 +0000 Subject: [PATCH 60/77] dev container: set the root path for rust analyzer Signed-off-by: Pedro Ferreira --- .devcontainer/almalinux/devcontainer.json | 5 ++++- .devcontainer/amazonlinux/devcontainer.json | 5 ++++- .devcontainer/debiantrixie/devcontainer.json | 5 ++++- .devcontainer/ubuntu20/devcontainer.json | 5 ++++- .devcontainer/ubuntu22/devcontainer.json | 5 ++++- .devcontainer/ubuntu24/devcontainer.json | 5 ++++- .devcontainer/ubuntu25/devcontainer.json | 5 ++++- 7 files changed, 28 insertions(+), 7 deletions(-) diff --git a/.devcontainer/almalinux/devcontainer.json b/.devcontainer/almalinux/devcontainer.json index 252bbf680..72c16e5c3 100644 --- a/.devcontainer/almalinux/devcontainer.json +++ b/.devcontainer/almalinux/devcontainer.json @@ -38,7 +38,10 @@ "editor.formatOnSave": true, "[cpp]": { "editor.defaultFormatter": "xaver.clang-format" - } + }, + "rust-analyzer.linkedProjects": [ + "${workspaceFolder}/rust/Cargo.toml" + ] } } }, diff --git a/.devcontainer/amazonlinux/devcontainer.json b/.devcontainer/amazonlinux/devcontainer.json index 2610a3e6e..9a12fd37e 100644 --- a/.devcontainer/amazonlinux/devcontainer.json +++ b/.devcontainer/amazonlinux/devcontainer.json @@ -42,7 +42,10 @@ "editor.formatOnSave": true, "[cpp]": { "editor.defaultFormatter": "xaver.clang-format" - } + }, + "rust-analyzer.linkedProjects": [ + "${workspaceFolder}/rust/Cargo.toml" + ] } } }, diff --git a/.devcontainer/debiantrixie/devcontainer.json b/.devcontainer/debiantrixie/devcontainer.json index 684157d1b..c3085b13e 100644 --- a/.devcontainer/debiantrixie/devcontainer.json +++ b/.devcontainer/debiantrixie/devcontainer.json @@ -38,7 +38,10 @@ "editor.formatOnSave": true, "[cpp]": { "editor.defaultFormatter": "xaver.clang-format" - } + }, + "rust-analyzer.linkedProjects": [ + "${workspaceFolder}/rust/Cargo.toml" + ] } } }, diff --git a/.devcontainer/ubuntu20/devcontainer.json b/.devcontainer/ubuntu20/devcontainer.json index e77831844..cb10c067c 100644 --- a/.devcontainer/ubuntu20/devcontainer.json +++ b/.devcontainer/ubuntu20/devcontainer.json @@ -40,7 +40,10 @@ "editor.formatOnSave": true, "[cpp]": { "editor.defaultFormatter": "xaver.clang-format" - } + }, + "rust-analyzer.linkedProjects": [ + "${workspaceFolder}/rust/Cargo.toml" + ] } } }, diff --git a/.devcontainer/ubuntu22/devcontainer.json b/.devcontainer/ubuntu22/devcontainer.json index 02c77d4d7..fdc217ecb 100644 --- a/.devcontainer/ubuntu22/devcontainer.json +++ b/.devcontainer/ubuntu22/devcontainer.json @@ -40,7 +40,10 @@ "editor.formatOnSave": true, "[cpp]": { "editor.defaultFormatter": "xaver.clang-format" - } + }, + "rust-analyzer.linkedProjects": [ + "${workspaceFolder}/rust/Cargo.toml" + ] } } }, diff --git a/.devcontainer/ubuntu24/devcontainer.json b/.devcontainer/ubuntu24/devcontainer.json index 52f08331e..57789ef23 100644 --- a/.devcontainer/ubuntu24/devcontainer.json +++ b/.devcontainer/ubuntu24/devcontainer.json @@ -40,7 +40,10 @@ "editor.formatOnSave": true, "[cpp]": { "editor.defaultFormatter": "xaver.clang-format" - } + }, + "rust-analyzer.linkedProjects": [ + "${workspaceFolder}/rust/Cargo.toml" + ] } } }, diff --git a/.devcontainer/ubuntu25/devcontainer.json b/.devcontainer/ubuntu25/devcontainer.json index 87ea80b54..d69f9d4b5 100644 --- a/.devcontainer/ubuntu25/devcontainer.json +++ b/.devcontainer/ubuntu25/devcontainer.json @@ -40,7 +40,10 @@ "editor.formatOnSave": true, "[cpp]": { "editor.defaultFormatter": "xaver.clang-format" - } + }, + "rust-analyzer.linkedProjects": [ + "${workspaceFolder}/rust/Cargo.toml" + ] } } }, From 56982562f63c75331e27012e17372284da176b69 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 15:00:49 +0000 Subject: [PATCH 61/77] rust: fix clippy lints Signed-off-by: Pedro Ferreira --- rust/mxl/examples/flow-reader.rs | 18 ++++++++---------- rust/mxl/examples/flow-writer.rs | 20 ++++++++++---------- rust/mxl/tests/basic_tests.rs | 6 +++--- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/rust/mxl/examples/flow-reader.rs b/rust/mxl/examples/flow-reader.rs index 60987c878..7f0e89eb2 100644 --- a/rust/mxl/examples/flow-reader.rs +++ b/rust/mxl/examples/flow-reader.rs @@ -95,17 +95,15 @@ fn read_samples( ); } batch_size as usize - } else { - if continous_flow_info.commitBatchSize == 0 { - let batch_size = (sample_rate.numerator / (100 * sample_rate.denominator)) as usize; - warn!( - "Writer batch size not available, using fallback value of {}.", - batch_size - ); + } else if continous_flow_info.commitBatchSize == 0 { + let batch_size = (sample_rate.numerator / (100 * sample_rate.denominator)) as usize; + warn!( + "Writer batch size not available, using fallback value of {}.", batch_size - } else { - continous_flow_info.commitBatchSize as usize - } + ); + batch_size + } else { + continous_flow_info.commitBatchSize as usize }; let mut read_head = reader.get_info()?.continuous_flow_info()?.headIndex; let mut read_head_valid_at = mxl_instance.get_time(); diff --git a/rust/mxl/examples/flow-writer.rs b/rust/mxl/examples/flow-writer.rs index 7f369c80b..440312124 100644 --- a/rust/mxl/examples/flow-writer.rs +++ b/rust/mxl/examples/flow-writer.rs @@ -88,8 +88,8 @@ pub fn write_grains( let mut grain_writer_access = writer.open_grain(grain_index)?; let payload = grain_writer_access.payload_mut(); let payload_len = payload.len(); - for i in 0..payload_len { - payload[i] = ((i as u64 + grain_index) % 256) as u8; + for (i, byte) in payload.iter_mut().enumerate() { + *byte = ((i as u64 + grain_index) % 256) as u8; } grain_writer_access.commit(payload_len as u32)?; @@ -130,10 +130,10 @@ pub fn write_samples( let mut remaining_samples = sample_count; loop { - if let Some(count) = remaining_samples { - if count == 0 { - break; - } + if let Some(count) = remaining_samples + && count == 0 + { + break; } let samples_to_write = u64::min(batch_size, remaining_samples.unwrap_or(u64::MAX)); if let Some(count) = remaining_samples { @@ -144,12 +144,12 @@ pub fn write_samples( let mut writing_sample_index = samples_index - batch_size + 1; for channel in 0..samples_write_access.channels() { let (data_1, data_2) = samples_write_access.channel_data_mut(channel)?; - for i in 0..data_1.len() { - data_1[i] = (writing_sample_index % 256) as u8; + for sample in data_1.iter_mut() { + *sample = (writing_sample_index % 256) as u8; writing_sample_index += 1; } - for i in 0..data_2.len() { - data_2[i] = (writing_sample_index % 256) as u8; + for sample in data_2.iter_mut() { + *sample = (writing_sample_index % 256) as u8; writing_sample_index += 1; } } diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index 003952a32..c153ce8b8 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -42,7 +42,8 @@ fn setup_test(test: &str) -> mxl::MxlInstance { fn read_flow_def>(path: P) -> String { let flow_config_file = mxl::config::get_mxl_repo_root().join(path); - let flow_def = std::fs::read_to_string(flow_config_file.as_path()) + + std::fs::read_to_string(flow_config_file.as_path()) .map_err(|error| { mxl::Error::Other(format!( "Error while reading flow definition from \"{}\": {}", @@ -50,8 +51,7 @@ fn read_flow_def>(path: P) -> String { error )) }) - .unwrap(); - flow_def + .unwrap() } fn prepare_flow_info>( From 380336dd21b70362be1132a9deb24ce65b43cc44 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 15:01:15 +0000 Subject: [PATCH 62/77] rust: add license to nextest config Signed-off-by: Pedro Ferreira --- rust/.config/nextest.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rust/.config/nextest.toml b/rust/.config/nextest.toml index 6a109e171..d7db88deb 100644 --- a/rust/.config/nextest.toml +++ b/rust/.config/nextest.toml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 + [profile.ci] fail-fast = false From d351b77bd19c8e80f694f50575f6a52cae062462 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 16:57:19 +0000 Subject: [PATCH 63/77] rust: upgrade tracing-subscriber cargo audit complained about RUSTSEC-2025-0055 Signed-off-by: Pedro Ferreira --- rust/Cargo.lock | 137 +++++++++++++++++++++++++++++++++--------------- rust/Cargo.toml | 2 +- 2 files changed, 95 insertions(+), 44 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index eda5e2d4f..9ea4d3b8b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -221,7 +221,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets", + "windows-targets 0.53.2", ] [[package]] @@ -232,11 +232,11 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] @@ -284,12 +284,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" dependencies = [ - "overload", - "winapi", + "windows-sys", ] [[package]] @@ -298,12 +297,6 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "pin-project-lite" version = "0.2.16" @@ -346,17 +339,8 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", + "regex-automata", + "regex-syntax", ] [[package]] @@ -367,15 +351,9 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax", ] -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" version = "0.8.5" @@ -501,14 +479,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "sharded-slab", "smallvec", "thread_local", @@ -625,64 +603,137 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + [[package]] name = "windows-targets" version = "0.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_gnullvm" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + [[package]] name = "windows_aarch64_msvc" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + [[package]] name = "windows_i686_gnu" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + [[package]] name = "windows_i686_gnullvm" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + [[package]] name = "windows_i686_msvc" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + [[package]] name = "windows_x86_64_gnu" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + [[package]] name = "windows_x86_64_gnullvm" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "windows_x86_64_msvc" version = "0.53.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index d986b3aa9..3a030638c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -20,7 +20,7 @@ dlopen2 = "0.8" futures = "0.3" thiserror = "2.0.12" tracing = { version = "0.1", features = ["log"] } -tracing-subscriber = { version = "0.3", features = ["env-filter", "std"] } +tracing-subscriber = { version = "0.3.20", features = ["env-filter", "std"] } uuid = { version = "1.17" } [workspace.dependencies.clap] From fd268c6727d066fc0abafa8bea5222c76214e315 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 16:58:28 +0000 Subject: [PATCH 64/77] rust: add version to Cargo.toml Signed-off-by: Pedro Ferreira --- rust/mxl-sys/Cargo.toml | 1 + rust/mxl/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/rust/mxl-sys/Cargo.toml b/rust/mxl-sys/Cargo.toml index 58e1dbf19..1da422524 100644 --- a/rust/mxl-sys/Cargo.toml +++ b/rust/mxl-sys/Cargo.toml @@ -6,6 +6,7 @@ name = "mxl-sys" edition.workspace = true publish.workspace = true version.workspace = true +license.workspace = true [dependencies] diff --git a/rust/mxl/Cargo.toml b/rust/mxl/Cargo.toml index 325c59ecf..ca6404890 100644 --- a/rust/mxl/Cargo.toml +++ b/rust/mxl/Cargo.toml @@ -6,6 +6,7 @@ name = "mxl" edition.workspace = true publish.workspace = true version.workspace = true +license.workspace = true [dependencies] mxl-sys = { path = "../mxl-sys" } From 28d648f7da63387682754d53867e85fef690a4c2 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 17:06:24 +0000 Subject: [PATCH 65/77] rust: add a config file for cargo deny Asked on Slack for guidance regarding which licenses should be accepted. Signed-off-by: Pedro Ferreira --- rust/deny.toml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 rust/deny.toml diff --git a/rust/deny.toml b/rust/deny.toml new file mode 100644 index 000000000..d8a4044e3 --- /dev/null +++ b/rust/deny.toml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project. +# SPDX-License-Identifier: Apache-2.0 + +# This section is considered when running `cargo deny check licenses` +# More documentation for the licenses section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html +[licenses] +# List of explicitly allowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +allow = [ + "MIT", + "Apache-2.0", + "Unicode-3.0", + "BSD-3-Clause", + "ISC" +] From 9b5cf0884e13ed4b5da471dfe9fafd64255fa284 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 14:33:02 +0100 Subject: [PATCH 66/77] ci: create the rust target directory Signed-off-by: Pedro Ferreira --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9a0332d88..79879a97f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,6 +87,9 @@ jobs: mkdir -p ${{ github.workspace }}/install chmod 777 ${{ github.workspace }}/install chmod g+s ${{ github.workspace }}/install + mkdir -p ${{ github.workspace }}/rust/target + chmod 777 ${{ github.workspace }}/rust/target + chmod g+s ${{ github.workspace }}/rust/target - name: Build Docker image if: steps.check-image.outputs.exists == 'false' From 24c2e9c4e09e4f19b2689fbca634a0815398209c Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 16:17:07 +0000 Subject: [PATCH 67/77] ci: add rust tasks Signed-off-by: Pedro Ferreira --- .github/workflows/build.yml | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 79879a97f..123008a4c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -165,6 +165,53 @@ jobs: ctest --output-junit test-results.xml " + - name: Cache Cargo registry + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-lint-${{ hashFiles('**/Cargo.toml') }} + restore-keys: ${{ runner.os }}-cargo-lint- + + - name: Check the Rust bindings + env: + RUSTFLAGS: "-Dwarnings" + run: | + docker run --mount src=${{ github.workspace }},target=/workspace/mxl,type=bind \ + -i mxl_build_container_with_source \ + bash -c " + cd /workspace/mxl/rust && \ + cargo fmt -- --check && \ + cargo clippy --all-targets --all-features --locked -- -D warnings && \ + cargo audit && \ + cargo outdated && \ + cargo machete && \ + cargo deny check all + " + + - name: Build Rust bindings + run: | + docker run --mount src=${{ github.workspace }},target=/workspace/mxl,type=bind \ + -i mxl_build_container_with_source \ + bash -c " + cd /workspace/mxl/rust && \ + cargo build --all-targets --locked && \ + cargo build --release --all-targets --locked + " + + - name: Test the Rust bindings + run: | + docker run --mount src=${{ github.workspace }},target=/workspace/mxl,type=bind \ + -i mxl_build_container_with_source \ + bash -c " + cd /workspace/mxl/rust && \ + cargo nextest run --release --locked + " + - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action@v2 if: always() From 752b6c1f2720de04ce32dfec77aabb6811e2b26e Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 17:05:13 +0000 Subject: [PATCH 68/77] ci: delete the original rust workflow Signed-off-by: Pedro Ferreira --- .github/workflows/rust.yml | 146 ------------------------------------- 1 file changed, 146 deletions(-) delete mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index a2fa3889b..000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project. -# SPDX-License-Identifier: Apache-2.0 - -name: Test the rust bindings - -on: - pull_request: - workflow_dispatch: - workflow_call: # We would like this to be called by the main job - -jobs: - dependencies: - name: Check Rust dependencies - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./rust - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Cache - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-deps-${{ hashFiles('**/Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo-deps- - - name: Use Rust stable 1.88 - uses: dtolnay/rust-toolchain@1.88.0 - - name: Install cargo audit - uses: baptiste0928/cargo-install@v3 - with: - crate: cargo-audit - - name: Install cargo outdated - uses: baptiste0928/cargo-install@v3 - with: - crate: cargo-outdated - - name: Install cargo udeps - uses: baptiste0928/cargo-install@v3 - with: - crate: cargo-udeps - - name: Install cargo deny - uses: baptiste0928/cargo-install@v3 - with: - crate: cargo-deny - - name: Audit dependencies - run: cargo audit -D warnings - - name: Outdated dependencies - run: cargo outdated -d 1 -w --exit-code 1 - - name: Install nightly-2025-06-26 # Same date as 1.88 release - uses: dtolnay/rust-toolchain@nightly - with: - toolchain: nightly-2025-06-26 - - name: Unused depedency check - run: cargo udeps --all-targets - - lint: - name: Perform Rust linting and documentation - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./rust - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Cache - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-lint-${{ hashFiles('**/Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo-lint- - - name: Use Rust stable 1.88 - uses: dtolnay/rust-toolchain@1.88.0 - with: - components: rustfmt, clippy - - name: build - run: cargo build --locked - - name: Format - run: cargo fmt -- --check - - name: Docs - run: cargo doc --all-features - env: - RUSTDOCFLAGS: "-D warnings" - - name: Clippy - run: cargo clippy --all-targets -F mxl-not-built -- -D warnings - - tests: - name: Run the tests - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./rust - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Cache - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-tests-${{ hashFiles('**/Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo-tests- - - name: Use Rust stable 1.88 - uses: dtolnay/rust-toolchain@1.88.0 - with: - components: rustfmt, clippy - - name: build - run: cargo build --locked -F mxl-not-built - - name: Test - run: cargo test --locked -F mxl-not-built - - name: Coverage - run: > - cargo llvm-cov --ignore-filename-regex "build.rs|ffi.rs|(.*)_test.rs" - --lcov --output-path lcov.info - - name: Report Coverage - uses: romeovs/lcov-reporter-action@v0.4.0 - if: ${{ github.event_name == 'pull_request' }} - with: - lcov-file: ./lcov.info - github-token: ${{ secrets.GITHUB_TOKEN }} - delete-old-comments: true From b053dfb232f1aec94f28165fa20188e9a71ff95b Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 19:04:12 +0000 Subject: [PATCH 69/77] rust: allow reusing a loaded api The api is now returned as an Arc, which can be cloned. Signed-off-by: Pedro Ferreira --- rust/mxl/src/api.rs | 10 +++++++--- rust/mxl/src/instance.rs | 8 +++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/rust/mxl/src/api.rs b/rust/mxl/src/api.rs index c47c18d03..fde080d53 100644 --- a/rust/mxl/src/api.rs +++ b/rust/mxl/src/api.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2025 2025 Contributors to the Media eXchange Layer project. // SPDX-License-Identifier: Apache-2.0 -use std::path::Path; +use std::{path::Path, sync::Arc}; use dlopen2::wrapper::{Container, WrapperApi}; @@ -154,6 +154,10 @@ pub struct MxlApi { mxl_get_time: unsafe extern "C" fn() -> u64, } -pub fn load_api(path_to_so_file: impl AsRef) -> Result> { - Ok(unsafe { Container::load(path_to_so_file.as_ref().as_os_str()) }?) +pub type MxlApiHandle = Arc>; + +pub fn load_api(path_to_so_file: impl AsRef) -> Result { + Ok(Arc::new(unsafe { + Container::load(path_to_so_file.as_ref().as_os_str()) + }?)) } diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index 64fd5d2a3..cfb2f15bc 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -3,16 +3,14 @@ use std::{ffi::CString, sync::Arc}; -use dlopen2::wrapper::Container; - -use crate::{Error, FlowInfo, MxlApi, MxlFlowReader, MxlFlowWriter, Result}; +use crate::{Error, FlowInfo, MxlFlowReader, MxlFlowWriter, Result, api::MxlApiHandle}; /// This struct stores the context that is shared by all objects. /// It is separated out from `MxlInstance` so that it can be cloned /// and other objects' lifetimes be decoupled from the MxlInstance /// itself. pub(crate) struct InstanceContext { - pub(crate) api: Container, + pub(crate) api: MxlApiHandle, pub(crate) instance: mxl_sys::mxlInstance, } @@ -70,7 +68,7 @@ pub struct MxlInstance { } impl MxlInstance { - pub fn new(api: Container, domain: &str, options: &str) -> Result { + pub fn new(api: MxlApiHandle, domain: &str, options: &str) -> Result { let instance = unsafe { api.mxl_create_instance( CString::new(domain)?.as_ptr(), From bc6e3dceb8341b07440bd62032918d04c1908a17 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 1 Oct 2025 19:36:00 +0000 Subject: [PATCH 70/77] rust: add non-blocking version of get grain Signed-off-by: Pedro Ferreira --- rust/mxl/src/grain/data.rs | 9 +++++++- rust/mxl/src/grain/reader.rs | 42 +++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/rust/mxl/src/grain/data.rs b/rust/mxl/src/grain/data.rs index d8cd98797..8ced646d5 100644 --- a/rust/mxl/src/grain/data.rs +++ b/rust/mxl/src/grain/data.rs @@ -2,8 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 pub struct GrainData<'a> { - pub user_data: &'a [u8], + /// The grain payload. This may be a partial payload if the grain is not complete. + /// The length of this slice is given by `commitedSize` in `mxlGrainInfo`. pub payload: &'a [u8], + + /// The total size of the grain payload, which may be larger than `payload.len()` if the grain is partial. + pub total_size: usize, + + /// The grain user data. The length of this slice is given by `userDataSize` in `mxlGrainInfo`. + pub user_data: &'a [u8], } impl<'a> GrainData<'a> { diff --git a/rust/mxl/src/grain/reader.rs b/rust/mxl/src/grain/reader.rs index df892e025..cc8d5ffbb 100644 --- a/rust/mxl/src/grain/reader.rs +++ b/rust/mxl/src/grain/reader.rs @@ -70,7 +70,47 @@ impl GrainReader { let payload = unsafe { std::slice::from_raw_parts(payload_ptr, grain_info.grainSize as usize) }; - Ok(GrainData { user_data, payload }) + Ok(GrainData { + user_data, + payload, + total_size: grain_info.grainSize as usize, + }) + } + + /// Non-blocking version of `get_complete_grain`. If the grain is not available, returns an error. + /// If the grain is partial, it is returned as is and the payload length will be smaller than the total grain size. + pub fn get_grain_non_blocking<'a>(&'a self, index: u64) -> Result> { + let mut grain_info: mxl_sys::mxlGrainInfo = unsafe { std::mem::zeroed() }; + let mut payload_ptr: *mut u8 = std::ptr::null_mut(); + unsafe { + Error::from_status(self.context.api.mxl_flow_reader_get_grain_non_blocking( + self.reader, + index, + &mut grain_info, + &mut payload_ptr, + ))?; + } + + if payload_ptr.is_null() { + return Err(Error::Other(format!( + "Failed to get grain payload for index {index}.", + ))); + } + + // SAFETY + // We know that the lifetime is as long as the flow, so it is at least self's lifetime. + // It may happen that the buffer is overwritten by a subsequent write, but it is safe. + let user_data: &'a [u8] = + unsafe { std::mem::transmute::<&[u8], &'a [u8]>(&grain_info.userData) }; + + let payload = + unsafe { std::slice::from_raw_parts(payload_ptr, grain_info.grainSize as usize) }; + + Ok(GrainData { + user_data, + payload, + total_size: grain_info.grainSize as usize, + }) } fn destroy_inner(&mut self) -> Result<()> { From ab8953b97475efa31d94a7763e49c303ec3b183a Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 2 Oct 2025 07:13:11 +0000 Subject: [PATCH 71/77] rust: rename MxlFlowReader to FlowReader Signed-off-by: Pedro Ferreira --- rust/mxl/src/flow/reader.rs | 8 ++++---- rust/mxl/src/instance.rs | 8 ++++---- rust/mxl/src/lib.rs | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/rust/mxl/src/flow/reader.rs b/rust/mxl/src/flow/reader.rs index 38abfd73c..2461c55b5 100644 --- a/rust/mxl/src/flow/reader.rs +++ b/rust/mxl/src/flow/reader.rs @@ -9,14 +9,14 @@ use crate::{ instance::InstanceContext, }; -pub struct MxlFlowReader { +pub struct FlowReader { context: Arc, reader: mxl_sys::mxlFlowReader, } /// The MXL readers and writers are not thread-safe, so we do not implement `Sync` for them, but /// there is no reason to not implement `Send`. -unsafe impl Send for MxlFlowReader {} +unsafe impl Send for FlowReader {} pub(crate) fn get_flow_info( context: &Arc, @@ -29,7 +29,7 @@ pub(crate) fn get_flow_info( Ok(FlowInfo { value: flow_info }) } -impl MxlFlowReader { +impl FlowReader { pub(crate) fn new(context: Arc, reader: mxl_sys::mxlFlowReader) -> Self { Self { context, reader } } @@ -65,7 +65,7 @@ impl MxlFlowReader { } } -impl Drop for MxlFlowReader { +impl Drop for FlowReader { fn drop(&mut self) { if !self.reader.is_null() && let Err(err) = Error::from_status(unsafe { diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index cfb2f15bc..b4ff8fcb4 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -3,7 +3,7 @@ use std::{ffi::CString, sync::Arc}; -use crate::{Error, FlowInfo, MxlFlowReader, MxlFlowWriter, Result, api::MxlApiHandle}; +use crate::{Error, FlowInfo, FlowReader, MxlFlowWriter, Result, api::MxlApiHandle}; /// This struct stores the context that is shared by all objects. /// It is separated out from `MxlInstance` so that it can be cloned @@ -44,7 +44,7 @@ impl Drop for InstanceContext { pub(crate) fn create_flow_reader( context: &Arc, flow_id: &str, -) -> Result { +) -> Result { let flow_id = CString::new(flow_id)?; let options = CString::new("")?; let mut reader: mxl_sys::mxlFlowReader = std::ptr::null_mut(); @@ -59,7 +59,7 @@ pub(crate) fn create_flow_reader( if reader.is_null() { return Err(Error::Other("Failed to create flow reader.".to_string())); } - Ok(MxlFlowReader::new(context.clone(), reader)) + Ok(FlowReader::new(context.clone(), reader)) } #[derive(Clone)] @@ -83,7 +83,7 @@ impl MxlInstance { } } - pub fn create_flow_reader(&self, flow_id: &str) -> Result { + pub fn create_flow_reader(&self, flow_id: &str) -> Result { create_flow_reader(&self.context, flow_id) } diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index 1b9ecbc9b..44db333cc 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -12,7 +12,7 @@ pub mod config; pub use api::{MxlApi, load_api}; pub use error::{Error, Result}; -pub use flow::{reader::MxlFlowReader, writer::MxlFlowWriter, *}; +pub use flow::{reader::FlowReader, writer::MxlFlowWriter, *}; pub use grain::{ data::*, reader::GrainReader, write_access::GrainWriteAccess, writer::GrainWriter, }; From 01a7bc3c263fb10d7ea1e68f9b86f61fabe3457b Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 2 Oct 2025 07:14:03 +0000 Subject: [PATCH 72/77] rust: rename MxlFlowWriter to FlowWriter Signed-off-by: Pedro Ferreira --- rust/mxl/src/flow/writer.rs | 8 ++++---- rust/mxl/src/instance.rs | 6 +++--- rust/mxl/src/lib.rs | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rust/mxl/src/flow/writer.rs b/rust/mxl/src/flow/writer.rs index b1234073d..0f7c6bca3 100644 --- a/rust/mxl/src/flow/writer.rs +++ b/rust/mxl/src/flow/writer.rs @@ -11,7 +11,7 @@ use crate::{ /// Generic MXL Flow Writer, which can be further used to build either the "discrete" (grain-based /// data like video frames or meta) or "continuous" (audio samples) flow writers in MXL terminology. -pub struct MxlFlowWriter { +pub struct FlowWriter { context: Arc, writer: mxl_sys::mxlFlowWriter, id: uuid::Uuid, @@ -19,9 +19,9 @@ pub struct MxlFlowWriter { /// The MXL readers and writers are not thread-safe, so we do not implement `Sync` for them, but /// there is no reason to not implement `Send`. -unsafe impl Send for MxlFlowWriter {} +unsafe impl Send for FlowWriter {} -impl MxlFlowWriter { +impl FlowWriter { pub(crate) fn new( context: Arc, writer: mxl_sys::mxlFlowWriter, @@ -77,7 +77,7 @@ impl MxlFlowWriter { } } -impl Drop for MxlFlowWriter { +impl Drop for FlowWriter { fn drop(&mut self) { if !self.writer.is_null() && let Err(err) = Error::from_status(unsafe { diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index b4ff8fcb4..e0d3374c3 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -3,7 +3,7 @@ use std::{ffi::CString, sync::Arc}; -use crate::{Error, FlowInfo, FlowReader, MxlFlowWriter, Result, api::MxlApiHandle}; +use crate::{Error, FlowInfo, FlowReader, FlowWriter, Result, api::MxlApiHandle}; /// This struct stores the context that is shared by all objects. /// It is separated out from `MxlInstance` so that it can be cloned @@ -87,7 +87,7 @@ impl MxlInstance { create_flow_reader(&self.context, flow_id) } - pub fn create_flow_writer(&self, flow_id: &str) -> Result { + pub fn create_flow_writer(&self, flow_id: &str) -> Result { let uuid = uuid::Uuid::parse_str(flow_id) .map_err(|_| Error::Other("Invalid flow ID format.".to_string()))?; let flow_id = CString::new(flow_id)?; @@ -104,7 +104,7 @@ impl MxlInstance { if writer.is_null() { return Err(Error::Other("Failed to create flow writer.".to_string())); } - Ok(MxlFlowWriter::new(self.context.clone(), writer, uuid)) + Ok(FlowWriter::new(self.context.clone(), writer, uuid)) } /// For now, we provide direct access to the MXL API for creating and diff --git a/rust/mxl/src/lib.rs b/rust/mxl/src/lib.rs index 44db333cc..d33137f77 100644 --- a/rust/mxl/src/lib.rs +++ b/rust/mxl/src/lib.rs @@ -12,7 +12,7 @@ pub mod config; pub use api::{MxlApi, load_api}; pub use error::{Error, Result}; -pub use flow::{reader::FlowReader, writer::MxlFlowWriter, *}; +pub use flow::{reader::FlowReader, writer::FlowWriter, *}; pub use grain::{ data::*, reader::GrainReader, write_access::GrainWriteAccess, writer::GrainWriter, }; From 6e31ff6762e8206d3fecfc2d2097aee643a07d4f Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 2 Oct 2025 12:08:12 +0000 Subject: [PATCH 73/77] rust: remove mxl prefix from api function names Signed-off-by: Pedro Ferreira --- rust/mxl/src/api.rs | 121 ++++++++++++++------------- rust/mxl/src/flow/reader.rs | 4 +- rust/mxl/src/flow/writer.rs | 2 +- rust/mxl/src/grain/reader.rs | 6 +- rust/mxl/src/grain/write_access.rs | 6 +- rust/mxl/src/grain/writer.rs | 4 +- rust/mxl/src/instance.rs | 34 ++++---- rust/mxl/src/samples/reader.rs | 4 +- rust/mxl/src/samples/write_access.rs | 6 +- rust/mxl/src/samples/writer.rs | 4 +- 10 files changed, 98 insertions(+), 93 deletions(-) diff --git a/rust/mxl/src/api.rs b/rust/mxl/src/api.rs index fde080d53..35f702540 100644 --- a/rust/mxl/src/api.rs +++ b/rust/mxl/src/api.rs @@ -10,148 +10,157 @@ use crate::Result; #[derive(WrapperApi)] pub struct MxlApi { #[dlopen2_name = "mxlGetVersion"] - mxl_get_version: + get_version: unsafe extern "C" fn(out_version: *mut mxl_sys::mxlVersionType) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] + #[dlopen2_name = "mxlCreateInstance"] - mxl_create_instance: unsafe extern "C" fn( - in_mxlDomain: *const std::os::raw::c_char, + create_instance: unsafe extern "C" fn( + in_mxl_domain: *const std::os::raw::c_char, in_options: *const std::os::raw::c_char, ) -> mxl_sys::mxlInstance, + #[dlopen2_name = "mxlGarbageCollectFlows"] - mxl_garbage_collect_flows: + garbage_collect_flows: unsafe extern "C" fn(in_instance: mxl_sys::mxlInstance) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlDestroyInstance"] - mxl_destroy_instance: - unsafe extern "C" fn(in_instance: mxl_sys::mxlInstance) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] + destroy_instance: unsafe extern "C" fn(in_instance: mxl_sys::mxlInstance) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlCreateFlow"] - mxl_create_flow: unsafe extern "C" fn( + create_flow: unsafe extern "C" fn( instance: mxl_sys::mxlInstance, - flowDef: *const std::os::raw::c_char, + flow_def: *const std::os::raw::c_char, options: *const std::os::raw::c_char, info: *mut mxl_sys::mxlFlowInfo, ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] + #[dlopen2_name = "mxlDestroyFlow"] - mxl_destroy_flow: unsafe extern "C" fn( + destroy_flow: unsafe extern "C" fn( instance: mxl_sys::mxlInstance, - flowId: *const std::os::raw::c_char, + flow_id: *const std::os::raw::c_char, ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] #[dlopen2_name = "mxlGetFlowDef"] - mxl_get_flow_def: unsafe extern "C" fn( + get_flow_def: unsafe extern "C" fn( instance: mxl_sys::mxlInstance, - flowId: *const ::std::os::raw::c_char, + flow_id: *const ::std::os::raw::c_char, buffer: *mut ::std::os::raw::c_char, - bufferSize: *mut usize, + buffer_size: *mut usize, ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] #[dlopen2_name = "mxlCreateFlowReader"] - mxl_create_flow_reader: unsafe extern "C" fn( + create_flow_reader: unsafe extern "C" fn( instance: mxl_sys::mxlInstance, - flowId: *const std::os::raw::c_char, + flow_id: *const std::os::raw::c_char, options: *const std::os::raw::c_char, reader: *mut mxl_sys::mxlFlowReader, ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlReleaseFlowReader"] - mxl_release_flow_reader: unsafe extern "C" fn( + release_flow_reader: unsafe extern "C" fn( instance: mxl_sys::mxlInstance, reader: mxl_sys::mxlFlowReader, ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] + #[dlopen2_name = "mxlCreateFlowWriter"] - mxl_create_flow_writer: unsafe extern "C" fn( + create_flow_writer: unsafe extern "C" fn( instance: mxl_sys::mxlInstance, - flowId: *const std::os::raw::c_char, + flow_id: *const std::os::raw::c_char, options: *const std::os::raw::c_char, writer: *mut mxl_sys::mxlFlowWriter, ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlReleaseFlowWriter"] - mxl_release_flow_writer: unsafe extern "C" fn( + release_flow_writer: unsafe extern "C" fn( instance: mxl_sys::mxlInstance, writer: mxl_sys::mxlFlowWriter, ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowReaderGetInfo"] - mxl_flow_reader_get_info: unsafe extern "C" fn( + flow_reader_get_info: unsafe extern "C" fn( reader: mxl_sys::mxlFlowReader, info: *mut mxl_sys::mxlFlowInfo, ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowReaderGetGrain"] - mxl_flow_reader_get_grain: unsafe extern "C" fn( + flow_reader_get_grain: unsafe extern "C" fn( reader: mxl_sys::mxlFlowReader, index: u64, - timeoutNs: u64, + timeout_ns: u64, grain: *mut mxl_sys::mxlGrainInfo, payload: *mut *mut u8, ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowReaderGetGrainNonBlocking"] - mxl_flow_reader_get_grain_non_blocking: unsafe extern "C" fn( + flow_reader_get_grain_non_blocking: unsafe extern "C" fn( reader: mxl_sys::mxlFlowReader, index: u64, grain: *mut mxl_sys::mxlGrainInfo, payload: *mut *mut u8, ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowWriterOpenGrain"] - mxl_flow_writer_open_grain: unsafe extern "C" fn( + flow_writer_open_grain: unsafe extern "C" fn( writer: mxl_sys::mxlFlowWriter, index: u64, - grainInfo: *mut mxl_sys::mxlGrainInfo, + grain_info: *mut mxl_sys::mxlGrainInfo, payload: *mut *mut u8, ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCancelGrain"] - mxl_flow_writer_cancel_grain: + flow_writer_cancel_grain: unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCommitGrain"] - mxl_flow_writer_commit_grain: unsafe extern "C" fn( + flow_writer_commit_grain: unsafe extern "C" fn( writer: mxl_sys::mxlFlowWriter, grain: *const mxl_sys::mxlGrainInfo, ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowReaderGetSamples"] - mxl_flow_reader_get_samples: unsafe extern "C" fn( + flow_reader_get_samples: unsafe extern "C" fn( reader: mxl_sys::mxlFlowReader, index: u64, count: usize, - payloadBuffersSlices: *mut mxl_sys::mxlWrappedMultiBufferSlice, + payload_buffers_slices: *mut mxl_sys::mxlWrappedMultiBufferSlice, ) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] + #[dlopen2_name = "mxlFlowWriterOpenSamples"] - mxl_flow_writer_open_samples: unsafe extern "C" fn( + flow_writer_open_samples: unsafe extern "C" fn( writer: mxl_sys::mxlFlowWriter, index: u64, count: usize, - payloadBuffersSlices: *mut mxl_sys::mxlMutableWrappedMultiBufferSlice, + payload_buffers_slices: *mut mxl_sys::mxlMutableWrappedMultiBufferSlice, ) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCancelSamples"] - mxl_flow_writer_cancel_samples: + flow_writer_cancel_samples: unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, + #[dlopen2_name = "mxlFlowWriterCommitSamples"] - mxl_flow_writer_commit_samples: + flow_writer_commit_samples: unsafe extern "C" fn(writer: mxl_sys::mxlFlowWriter) -> mxl_sys::mxlStatus, - #[allow(non_snake_case)] + #[dlopen2_name = "mxlGetCurrentIndex"] - mxl_get_current_index: unsafe extern "C" fn(editRate: *const mxl_sys::mxlRational) -> u64, - #[allow(non_snake_case)] + get_current_index: unsafe extern "C" fn(edit_rate: *const mxl_sys::mxlRational) -> u64, + #[dlopen2_name = "mxlGetNsUntilIndex"] - mxl_get_ns_until_index: - unsafe extern "C" fn(index: u64, editRate: *const mxl_sys::mxlRational) -> u64, - #[allow(non_snake_case)] + get_ns_until_index: + unsafe extern "C" fn(index: u64, edit_rate: *const mxl_sys::mxlRational) -> u64, + #[dlopen2_name = "mxlTimestampToIndex"] - mxl_timestamp_to_index: - unsafe extern "C" fn(editRate: *const mxl_sys::mxlRational, timestamp: u64) -> u64, - #[allow(non_snake_case)] + timestamp_to_index: + unsafe extern "C" fn(edit_rate: *const mxl_sys::mxlRational, timestamp: u64) -> u64, + #[dlopen2_name = "mxlIndexToTimestamp"] - mxl_index_to_timestamp: - unsafe extern "C" fn(editRate: *const mxl_sys::mxlRational, index: u64) -> u64, + index_to_timestamp: + unsafe extern "C" fn(edit_rate: *const mxl_sys::mxlRational, index: u64) -> u64, + #[dlopen2_name = "mxlSleepForNs"] - mxl_sleep_for_ns: unsafe extern "C" fn(ns: u64), + sleep_for_ns: unsafe extern "C" fn(ns: u64), + #[dlopen2_name = "mxlGetTime"] - mxl_get_time: unsafe extern "C" fn() -> u64, + get_time: unsafe extern "C" fn() -> u64, } pub type MxlApiHandle = Arc>; diff --git a/rust/mxl/src/flow/reader.rs b/rust/mxl/src/flow/reader.rs index 2461c55b5..522f0eedc 100644 --- a/rust/mxl/src/flow/reader.rs +++ b/rust/mxl/src/flow/reader.rs @@ -24,7 +24,7 @@ pub(crate) fn get_flow_info( ) -> Result { let mut flow_info: mxl_sys::mxlFlowInfo = unsafe { std::mem::zeroed() }; unsafe { - Error::from_status(context.api.mxl_flow_reader_get_info(reader, &mut flow_info))?; + Error::from_status(context.api.flow_reader_get_info(reader, &mut flow_info))?; } Ok(FlowInfo { value: flow_info }) } @@ -71,7 +71,7 @@ impl Drop for FlowReader { && let Err(err) = Error::from_status(unsafe { self.context .api - .mxl_release_flow_reader(self.context.instance, self.reader) + .release_flow_reader(self.context.instance, self.reader) }) { tracing::error!("Failed to release MXL flow reader: {:?}", err); diff --git a/rust/mxl/src/flow/writer.rs b/rust/mxl/src/flow/writer.rs index 0f7c6bca3..1b80de583 100644 --- a/rust/mxl/src/flow/writer.rs +++ b/rust/mxl/src/flow/writer.rs @@ -83,7 +83,7 @@ impl Drop for FlowWriter { && let Err(err) = Error::from_status(unsafe { self.context .api - .mxl_release_flow_writer(self.context.instance, self.writer) + .release_flow_writer(self.context.instance, self.writer) }) { tracing::error!("Failed to release MXL flow writer: {:?}", err); diff --git a/rust/mxl/src/grain/reader.rs b/rust/mxl/src/grain/reader.rs index cc8d5ffbb..e3d2946c3 100644 --- a/rust/mxl/src/grain/reader.rs +++ b/rust/mxl/src/grain/reader.rs @@ -41,7 +41,7 @@ impl GrainReader { let timeout_ns = timeout.as_nanos() as u64; loop { unsafe { - Error::from_status(self.context.api.mxl_flow_reader_get_grain( + Error::from_status(self.context.api.flow_reader_get_grain( self.reader, index, timeout_ns, @@ -83,7 +83,7 @@ impl GrainReader { let mut grain_info: mxl_sys::mxlGrainInfo = unsafe { std::mem::zeroed() }; let mut payload_ptr: *mut u8 = std::ptr::null_mut(); unsafe { - Error::from_status(self.context.api.mxl_flow_reader_get_grain_non_blocking( + Error::from_status(self.context.api.flow_reader_get_grain_non_blocking( self.reader, index, &mut grain_info, @@ -124,7 +124,7 @@ impl GrainReader { Error::from_status(unsafe { self.context .api - .mxl_release_flow_reader(self.context.instance, reader) + .release_flow_reader(self.context.instance, reader) }) } } diff --git a/rust/mxl/src/grain/write_access.rs b/rust/mxl/src/grain/write_access.rs index c34696086..25dcb1e51 100644 --- a/rust/mxl/src/grain/write_access.rs +++ b/rust/mxl/src/grain/write_access.rs @@ -70,7 +70,7 @@ impl<'a> GrainWriteAccess<'a> { Error::from_status( self.context .api - .mxl_flow_writer_commit_grain(self.writer, &self.grain_info), + .flow_writer_commit_grain(self.writer, &self.grain_info), ) } } @@ -82,7 +82,7 @@ impl<'a> GrainWriteAccess<'a> { pub fn cancel(mut self) -> Result<()> { self.committed_or_canceled = true; - unsafe { Error::from_status(self.context.api.mxl_flow_writer_cancel_grain(self.writer)) } + unsafe { Error::from_status(self.context.api.flow_writer_cancel_grain(self.writer)) } } } @@ -90,7 +90,7 @@ impl<'a> Drop for GrainWriteAccess<'a> { fn drop(&mut self) { if !self.committed_or_canceled && let Err(error) = unsafe { - Error::from_status(self.context.api.mxl_flow_writer_cancel_grain(self.writer)) + Error::from_status(self.context.api.flow_writer_cancel_grain(self.writer)) } { error!("Failed to cancel grain write on drop: {:?}", error); diff --git a/rust/mxl/src/grain/writer.rs b/rust/mxl/src/grain/writer.rs index 85e2598d6..e312627e1 100644 --- a/rust/mxl/src/grain/writer.rs +++ b/rust/mxl/src/grain/writer.rs @@ -34,7 +34,7 @@ impl GrainWriter { let mut grain_info: mxl_sys::mxlGrainInfo = unsafe { std::mem::zeroed() }; let mut payload_ptr: *mut u8 = std::ptr::null_mut(); unsafe { - Error::from_status(self.context.api.mxl_flow_writer_open_grain( + Error::from_status(self.context.api.flow_writer_open_grain( self.writer, index, &mut grain_info, @@ -67,7 +67,7 @@ impl GrainWriter { Error::from_status(unsafe { self.context .api - .mxl_release_flow_writer(self.context.instance, writer) + .release_flow_writer(self.context.instance, writer) }) } } diff --git a/rust/mxl/src/instance.rs b/rust/mxl/src/instance.rs index e0d3374c3..ae9a91107 100644 --- a/rust/mxl/src/instance.rs +++ b/rust/mxl/src/instance.rs @@ -27,7 +27,7 @@ impl InstanceContext { unsafe { let mut instance = std::ptr::null_mut(); std::mem::swap(&mut self.instance, &mut instance); - self.api.mxl_destroy_instance(self.instance) + self.api.destroy_instance(self.instance) }; Ok(()) } @@ -36,7 +36,7 @@ impl InstanceContext { impl Drop for InstanceContext { fn drop(&mut self) { if !self.instance.is_null() { - unsafe { self.api.mxl_destroy_instance(self.instance) }; + unsafe { self.api.destroy_instance(self.instance) }; } } } @@ -49,7 +49,7 @@ pub(crate) fn create_flow_reader( let options = CString::new("")?; let mut reader: mxl_sys::mxlFlowReader = std::ptr::null_mut(); unsafe { - Error::from_status(context.api.mxl_create_flow_reader( + Error::from_status(context.api.create_flow_reader( context.instance, flow_id.as_ptr(), options.as_ptr(), @@ -70,7 +70,7 @@ pub struct MxlInstance { impl MxlInstance { pub fn new(api: MxlApiHandle, domain: &str, options: &str) -> Result { let instance = unsafe { - api.mxl_create_instance( + api.create_instance( CString::new(domain)?.as_ptr(), CString::new(options)?.as_ptr(), ) @@ -94,7 +94,7 @@ impl MxlInstance { let options = CString::new("")?; let mut writer: mxl_sys::mxlFlowWriter = std::ptr::null_mut(); unsafe { - Error::from_status(self.context.api.mxl_create_flow_writer( + Error::from_status(self.context.api.create_flow_writer( self.context.instance, flow_id.as_ptr(), options.as_ptr(), @@ -116,7 +116,7 @@ impl MxlInstance { let mut info = std::mem::MaybeUninit::::uninit(); unsafe { - Error::from_status(self.context.api.mxl_create_flow( + Error::from_status(self.context.api.create_flow( self.context.instance, flow_def.as_ptr(), options.as_ptr(), @@ -135,7 +135,7 @@ impl MxlInstance { Error::from_status( self.context .api - .mxl_destroy_flow(self.context.instance, flow_id.as_ptr()), + .destroy_flow(self.context.instance, flow_id.as_ptr()), )?; } Ok(()) @@ -148,7 +148,7 @@ impl MxlInstance { let mut buffer_size = INITIAL_BUFFER_SIZE; let status = unsafe { - self.context.api.mxl_get_flow_def( + self.context.api.get_flow_def( self.context.instance, flow_id.as_ptr(), buffer.as_mut_ptr() as *mut std::os::raw::c_char, @@ -159,7 +159,7 @@ impl MxlInstance { if status == mxl_sys::MXL_ERR_INVALID_ARG && buffer_size > INITIAL_BUFFER_SIZE { buffer = vec![0; buffer_size]; unsafe { - Error::from_status(self.context.api.mxl_get_flow_def( + Error::from_status(self.context.api.get_flow_def( self.context.instance, flow_id.as_ptr(), buffer.as_mut_ptr() as *mut std::os::raw::c_char, @@ -180,7 +180,7 @@ impl MxlInstance { } pub fn get_current_index(&self, rational: &mxl_sys::mxlRational) -> u64 { - unsafe { self.context.api.mxl_get_current_index(rational) } + unsafe { self.context.api.get_current_index(rational) } } pub fn get_duration_until_index( @@ -188,7 +188,7 @@ impl MxlInstance { index: u64, rate: &mxl_sys::mxlRational, ) -> Result { - let duration_ns = unsafe { self.context.api.mxl_get_ns_until_index(index, rate) }; + let duration_ns = unsafe { self.context.api.get_ns_until_index(index, rate) }; if duration_ns == u64::MAX { Err(Error::Other(format!( "Failed to get duration until index, invalid rate {}/{}.", @@ -201,7 +201,7 @@ impl MxlInstance { /// TODO: Make timestamp a strong type. pub fn timestamp_to_index(&self, timestamp: u64, rate: &mxl_sys::mxlRational) -> Result { - let index = unsafe { self.context.api.mxl_timestamp_to_index(rate, timestamp) }; + let index = unsafe { self.context.api.timestamp_to_index(rate, timestamp) }; if index == u64::MAX { Err(Error::Other(format!( "Failed to convert timestamp to index, invalid rate {}/{}.", @@ -213,7 +213,7 @@ impl MxlInstance { } pub fn index_to_timestamp(&self, index: u64, rate: &mxl_sys::mxlRational) -> Result { - let timestamp = unsafe { self.context.api.mxl_index_to_timestamp(rate, index) }; + let timestamp = unsafe { self.context.api.index_to_timestamp(rate, index) }; if timestamp == u64::MAX { Err(Error::Other(format!( "Failed to convert index to timestamp, invalid rate {}/{}.", @@ -225,15 +225,11 @@ impl MxlInstance { } pub fn sleep_for(&self, duration: std::time::Duration) { - unsafe { - self.context - .api - .mxl_sleep_for_ns(duration.as_nanos() as u64) - } + unsafe { self.context.api.sleep_for_ns(duration.as_nanos() as u64) } } pub fn get_time(&self) -> u64 { - unsafe { self.context.api.mxl_get_time() } + unsafe { self.context.api.get_time() } } /// This function forces the destruction of the MXL instance. diff --git a/rust/mxl/src/samples/reader.rs b/rust/mxl/src/samples/reader.rs index 9c31d7e6a..4ad387c0d 100644 --- a/rust/mxl/src/samples/reader.rs +++ b/rust/mxl/src/samples/reader.rs @@ -34,7 +34,7 @@ impl SamplesReader { pub fn get_samples(&self, index: u64, count: usize) -> Result> { let mut buffer_slice: mxl_sys::mxlWrappedMultiBufferSlice = unsafe { std::mem::zeroed() }; unsafe { - Error::from_status(self.context.api.mxl_flow_reader_get_samples( + Error::from_status(self.context.api.flow_reader_get_samples( self.reader, index, count, @@ -55,7 +55,7 @@ impl SamplesReader { Error::from_status(unsafe { self.context .api - .mxl_release_flow_reader(self.context.instance, reader) + .release_flow_reader(self.context.instance, reader) }) } } diff --git a/rust/mxl/src/samples/write_access.rs b/rust/mxl/src/samples/write_access.rs index 5b43c53ea..536720369 100644 --- a/rust/mxl/src/samples/write_access.rs +++ b/rust/mxl/src/samples/write_access.rs @@ -40,7 +40,7 @@ impl<'a> SamplesWriteAccess<'a> { pub fn commit(mut self) -> crate::Result<()> { self.committed_or_canceled = true; - unsafe { Error::from_status(self.context.api.mxl_flow_writer_commit_samples(self.writer)) } + unsafe { Error::from_status(self.context.api.flow_writer_commit_samples(self.writer)) } } /// Please note that the behavior of canceling samples writing is dependent on the behavior @@ -50,7 +50,7 @@ impl<'a> SamplesWriteAccess<'a> { pub fn cancel(mut self) -> crate::Result<()> { self.committed_or_canceled = true; - unsafe { Error::from_status(self.context.api.mxl_flow_writer_cancel_samples(self.writer)) } + unsafe { Error::from_status(self.context.api.flow_writer_cancel_samples(self.writer)) } } pub fn channels(&self) -> usize { @@ -87,7 +87,7 @@ impl<'a> Drop for SamplesWriteAccess<'a> { fn drop(&mut self) { if !self.committed_or_canceled && let Err(error) = unsafe { - Error::from_status(self.context.api.mxl_flow_writer_cancel_samples(self.writer)) + Error::from_status(self.context.api.flow_writer_cancel_samples(self.writer)) } { error!("Failed to cancel grain write on drop: {:?}", error); diff --git a/rust/mxl/src/samples/writer.rs b/rust/mxl/src/samples/writer.rs index ecc455b47..c8f6996b5 100644 --- a/rust/mxl/src/samples/writer.rs +++ b/rust/mxl/src/samples/writer.rs @@ -28,7 +28,7 @@ impl SamplesWriter { let mut buffer_slice: mxl_sys::mxlMutableWrappedMultiBufferSlice = unsafe { std::mem::zeroed() }; unsafe { - Error::from_status(self.context.api.mxl_flow_writer_open_samples( + Error::from_status(self.context.api.flow_writer_open_samples( self.writer, index, count, @@ -53,7 +53,7 @@ impl SamplesWriter { Error::from_status(unsafe { self.context .api - .mxl_release_flow_writer(self.context.instance, writer) + .release_flow_writer(self.context.instance, writer) }) } } From 403ac4d3cf68e8d328eefcda0bdc2570e7fb9d06 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Tue, 14 Oct 2025 14:18:14 +0200 Subject: [PATCH 74/77] Only push docker images from pipeline if not building a fork - Forks do not have enough permissions to do so. Signed-off-by: Pavel Cernohorsky --- .github/workflows/build.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 123008a4c..026005a7c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -103,8 +103,19 @@ jobs: -f .devcontainer/Dockerfile \ .devcontainer + - name: Check if PR comes from a fork + id: is-fork-check + run: | + if [[ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]]; then + echo "This pull request originates from a forked repository." + echo "is-fork=true" >> "$GITHUB_OUTPUT" + else + echo "This pull request originates from the base repository." + echo "is-fork=false" >> "$GITHUB_OUTPUT" + fi + - name: Push Docker image to registry - if: steps.check-image.outputs.exists == 'false' + if: steps.check-image.outputs.exists == 'false' && steps.is-fork-check.outputs.is-fork == 'false' run: | docker push ${{ steps.docker-hash.outputs.image-tag }} From 87611b1415753373903757a96050d2d74e3bda7d Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Thu, 16 Oct 2025 15:28:45 +0200 Subject: [PATCH 75/77] Make Rust bindings compatiple with MXL 0.8.x Signed-off-by: Pavel Cernohorsky --- rust/mxl/examples/flow-reader.rs | 14 +++++++------- rust/mxl/examples/flow-writer.rs | 6 ++++-- rust/mxl/src/flow.rs | 4 ++++ rust/mxl/src/grain/reader.rs | 2 +- rust/mxl/src/grain/write_access.rs | 14 +++++++------- rust/mxl/tests/basic_tests.rs | 4 ++-- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/rust/mxl/examples/flow-reader.rs b/rust/mxl/examples/flow-reader.rs index 7f0e89eb2..d22907881 100644 --- a/rust/mxl/examples/flow-reader.rs +++ b/rust/mxl/examples/flow-reader.rs @@ -83,19 +83,19 @@ fn read_samples( ) -> Result<(), mxl::Error> { let flow_id = flow_info.common_flow_info().id().to_string(); let sample_rate = flow_info.continuous_flow_info()?.sampleRate; - let continous_flow_info = flow_info.continuous_flow_info()?; + let common_flow_info = flow_info.common_flow_info(); let batch_size = if let Some(batch_size) = batch_size { - if continous_flow_info.commitBatchSize != 0 - && batch_size != continous_flow_info.commitBatchSize as u64 + if common_flow_info.max_commit_batch_size_hint() != 0 + && batch_size != common_flow_info.max_commit_batch_size_hint() as u64 { warn!( "Writer batch size is set to {}, but sample batch size is provided, using the \ - latter.", - continous_flow_info.commitBatchSize + latter.", + common_flow_info.max_commit_batch_size_hint() ); } batch_size as usize - } else if continous_flow_info.commitBatchSize == 0 { + } else if common_flow_info.max_commit_batch_size_hint() == 0 { let batch_size = (sample_rate.numerator / (100 * sample_rate.denominator)) as usize; warn!( "Writer batch size not available, using fallback value of {}.", @@ -103,7 +103,7 @@ fn read_samples( ); batch_size } else { - continous_flow_info.commitBatchSize as usize + common_flow_info.max_commit_batch_size_hint() as usize }; let mut read_head = reader.get_info()?.continuous_flow_info()?.headIndex; let mut read_head_valid_at = mxl_instance.get_time(); diff --git a/rust/mxl/examples/flow-writer.rs b/rust/mxl/examples/flow-writer.rs index 440312124..6f38ffa35 100644 --- a/rust/mxl/examples/flow-writer.rs +++ b/rust/mxl/examples/flow-writer.rs @@ -86,17 +86,19 @@ pub fn write_grains( } let mut grain_writer_access = writer.open_grain(grain_index)?; + let total_slices = grain_writer_access.total_slices(); let payload = grain_writer_access.payload_mut(); let payload_len = payload.len(); for (i, byte) in payload.iter_mut().enumerate() { *byte = ((i as u64 + grain_index) % 256) as u8; } - grain_writer_access.commit(payload_len as u32)?; + grain_writer_access.commit(total_slices)?; let timestamp = mxl_instance.index_to_timestamp(grain_index + 1, &grain_rate)?; let sleep_duration = mxl_instance.get_duration_until_index(grain_index + 1, &grain_rate)?; info!( - "Finished writing {payload_len} bytes into grain {grain_index}, will sleep for {:?} until timestamp {timestamp}.", + "Finished writing {payload_len} bytes ({total_slices} slices) into grain {grain_index}, will sleep \ + for {:?} until timestamp {timestamp}.", sleep_duration ); grain_index += 1; diff --git a/rust/mxl/src/flow.rs b/rust/mxl/src/flow.rs index b91cd4f9c..acfc5bd79 100644 --- a/rust/mxl/src/flow.rs +++ b/rust/mxl/src/flow.rs @@ -75,4 +75,8 @@ impl CommonFlowInfo<'_> { pub fn id(&self) -> Uuid { Uuid::from_bytes(self.0.id) } + + pub fn max_commit_batch_size_hint(&self) -> u32 { + self.0.maxCommitBatchSizeHint + } } diff --git a/rust/mxl/src/grain/reader.rs b/rust/mxl/src/grain/reader.rs index e3d2946c3..c4a2958fc 100644 --- a/rust/mxl/src/grain/reader.rs +++ b/rust/mxl/src/grain/reader.rs @@ -49,7 +49,7 @@ impl GrainReader { &mut payload_ptr, ))?; } - if grain_info.commitedSize != grain_info.grainSize { + if grain_info.validSlices != grain_info.totalSlices { // We don't need partial grains. Wait for the grain to be complete. continue; } diff --git a/rust/mxl/src/grain/write_access.rs b/rust/mxl/src/grain/write_access.rs index 25dcb1e51..edf81fe63 100644 --- a/rust/mxl/src/grain/write_access.rs +++ b/rust/mxl/src/grain/write_access.rs @@ -51,20 +51,20 @@ impl<'a> GrainWriteAccess<'a> { self.grain_info.grainSize } - pub fn committed_size(&self) -> u32 { - self.grain_info.commitedSize + pub fn total_slices(&self) -> u16 { + self.grain_info.totalSlices } - pub fn commit(mut self, commited_size: u32) -> Result<()> { + pub fn commit(mut self, valid_slices: u16) -> Result<()> { self.committed_or_canceled = true; - if commited_size > self.grain_info.grainSize { + if valid_slices > self.grain_info.totalSlices { return Err(Error::Other(format!( - "Commited size {} cannot exceed grain size {}.", - commited_size, self.grain_info.grainSize + "Valid slices {} cannot exceed total slices {}.", + valid_slices, self.grain_info.totalSlices ))); } - self.grain_info.commitedSize = commited_size; + self.grain_info.validSlices = valid_slices; unsafe { Error::from_status( diff --git a/rust/mxl/tests/basic_tests.rs b/rust/mxl/tests/basic_tests.rs index c153ce8b8..aea4edcd4 100644 --- a/rust/mxl/tests/basic_tests.rs +++ b/rust/mxl/tests/basic_tests.rs @@ -74,8 +74,8 @@ fn basic_mxl_grain_writing_reading() { let rate = flow_info.discrete_flow_info().unwrap().grainRate; let current_index = mxl_instance.get_current_index(&rate); let grain_write_access = grain_writer.open_grain(current_index).unwrap(); - let grain_size = grain_write_access.max_size(); - grain_write_access.commit(grain_size).unwrap(); + let total_slices = grain_write_access.total_slices(); + grain_write_access.commit(total_slices).unwrap(); let grain_data = grain_reader .get_complete_grain(current_index, Duration::from_secs(5)) .unwrap(); From 8ad02c966de2d105605692b39b713b8b810acbe8 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Mon, 20 Oct 2025 16:07:36 +0200 Subject: [PATCH 76/77] Do not do debug build in Rust pipelines Signed-off-by: Pavel Cernohorsky --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 026005a7c..e39debf65 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -210,7 +210,6 @@ jobs: -i mxl_build_container_with_source \ bash -c " cd /workspace/mxl/rust && \ - cargo build --all-targets --locked && \ cargo build --release --all-targets --locked " From c56f54b80a36f29e225c7f559ab73671585b8a00 Mon Sep 17 00:00:00 2001 From: Pavel Cernohorsky Date: Fri, 31 Oct 2025 14:06:28 +0100 Subject: [PATCH 77/77] Make Rust bindings compatiple with the latest MXL Signed-off-by: Pavel Cernohorsky --- rust/mxl/src/grain/data.rs | 5 ----- rust/mxl/src/grain/reader.rs | 8 -------- rust/mxl/src/grain/write_access.rs | 4 ---- 3 files changed, 17 deletions(-) diff --git a/rust/mxl/src/grain/data.rs b/rust/mxl/src/grain/data.rs index 8ced646d5..fca910fac 100644 --- a/rust/mxl/src/grain/data.rs +++ b/rust/mxl/src/grain/data.rs @@ -8,9 +8,6 @@ pub struct GrainData<'a> { /// The total size of the grain payload, which may be larger than `payload.len()` if the grain is partial. pub total_size: usize, - - /// The grain user data. The length of this slice is given by `userDataSize` in `mxlGrainInfo`. - pub user_data: &'a [u8], } impl<'a> GrainData<'a> { @@ -26,14 +23,12 @@ impl<'a> AsRef> for GrainData<'a> { } pub struct OwnedGrainData { - pub user_data: Vec, pub payload: Vec, } impl<'a> From<&GrainData<'a>> for OwnedGrainData { fn from(value: &GrainData<'a>) -> Self { Self { - user_data: value.user_data.to_vec(), payload: value.payload.to_vec(), } } diff --git a/rust/mxl/src/grain/reader.rs b/rust/mxl/src/grain/reader.rs index c4a2958fc..ee07c32e7 100644 --- a/rust/mxl/src/grain/reader.rs +++ b/rust/mxl/src/grain/reader.rs @@ -64,14 +64,10 @@ impl GrainReader { // SAFETY // We know that the lifetime is as long as the flow, so it is at least self's lifetime. // It may happen that the buffer is overwritten by a subsequent write, but it is safe. - let user_data: &'a [u8] = - unsafe { std::mem::transmute::<&[u8], &'a [u8]>(&grain_info.userData) }; - let payload = unsafe { std::slice::from_raw_parts(payload_ptr, grain_info.grainSize as usize) }; Ok(GrainData { - user_data, payload, total_size: grain_info.grainSize as usize, }) @@ -100,14 +96,10 @@ impl GrainReader { // SAFETY // We know that the lifetime is as long as the flow, so it is at least self's lifetime. // It may happen that the buffer is overwritten by a subsequent write, but it is safe. - let user_data: &'a [u8] = - unsafe { std::mem::transmute::<&[u8], &'a [u8]>(&grain_info.userData) }; - let payload = unsafe { std::slice::from_raw_parts(payload_ptr, grain_info.grainSize as usize) }; Ok(GrainData { - user_data, payload, total_size: grain_info.grainSize as usize, }) diff --git a/rust/mxl/src/grain/write_access.rs b/rust/mxl/src/grain/write_access.rs index edf81fe63..7fd6aa63c 100644 --- a/rust/mxl/src/grain/write_access.rs +++ b/rust/mxl/src/grain/write_access.rs @@ -43,10 +43,6 @@ impl<'a> GrainWriteAccess<'a> { } } - pub fn user_data_mut(&mut self) -> &mut [u8] { - &mut self.grain_info.userData - } - pub fn max_size(&self) -> u32 { self.grain_info.grainSize }