diff --git a/Cargo.lock b/Cargo.lock index 5cdd6cd2..ae19a250 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -420,26 +420,6 @@ dependencies = [ "tower", ] -[[package]] -name = "bincode" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" -dependencies = [ - "bincode_derive", - "serde", - "unty", -] - -[[package]] -name = "bincode_derive" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" -dependencies = [ - "virtue", -] - [[package]] name = "bindgen" version = "0.72.1" @@ -2027,7 +2007,6 @@ dependencies = [ "anyhow", "async-trait", "bigtable_rs", - "bincode", "bytes", "data-encoding", "futures-util", @@ -3758,12 +3737,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "unty" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" - [[package]] name = "ureq" version = "3.1.2" @@ -3848,12 +3821,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "virtue" -version = "0.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" - [[package]] name = "want" version = "0.3.1" diff --git a/objectstore-service/Cargo.toml b/objectstore-service/Cargo.toml index dd88a9e3..896e66ad 100644 --- a/objectstore-service/Cargo.toml +++ b/objectstore-service/Cargo.toml @@ -13,7 +13,6 @@ publish = false anyhow = { workspace = true } async-trait = { workspace = true } bigtable_rs = "0.2.18" -bincode = { version = "2.0.1", features = ["serde"] } bytes = { workspace = true } data-encoding = "2.9.0" futures-util = { workspace = true } diff --git a/objectstore-service/src/backend/bigtable.rs b/objectstore-service/src/backend/bigtable.rs index 820fe191..eb5d43aa 100644 --- a/objectstore-service/src/backend/bigtable.rs +++ b/objectstore-service/src/backend/bigtable.rs @@ -13,8 +13,6 @@ use crate::backend::common::{Backend, BackendStream}; /// Connection timeout used for the initial connection to BigQuery. const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); -/// Config for bincode encoding and decoding. -const BC_CONFIG: bincode::config::Configuration = bincode::config::standard(); /// Time to debounce bumping an object with configured TTI. const TTI_DEBOUNCE: Duration = Duration::from_secs(24 * 3600); // 1 day @@ -23,7 +21,7 @@ const REQUEST_RETRY_COUNT: usize = 2; /// Column that stores the raw payload (compressed). const COLUMN_PAYLOAD: &[u8] = b"p"; -/// Column that stores metadata in bincode. +/// Column that stores metadata in JSON. const COLUMN_METADATA: &[u8] = b"m"; /// Column family that uses timestamp-based garbage collection. /// @@ -170,8 +168,7 @@ impl BigTableBackend { family_name: family.to_owned(), column_qualifier: COLUMN_METADATA.to_owned(), timestamp_micros, - // TODO: Do we really want bincode here? - value: bincode::serde::encode_to_vec(metadata, BC_CONFIG)?, + value: serde_json::to_vec(metadata).with_context(|| "failed to encode metadata")?, }), ]; self.mutate(path, mutations, action).await @@ -254,7 +251,8 @@ impl Backend for BigTableBackend { // TODO: Log if the timestamp is invalid. } self::COLUMN_METADATA => { - metadata = bincode::serde::decode_from_slice(&cell.value, BC_CONFIG)?.0; + metadata = serde_json::from_slice(&cell.value) + .with_context(|| "failed to decode metadata")?; } _ => { // TODO: Log unknown column diff --git a/objectstore-service/src/backend/local_fs.rs b/objectstore-service/src/backend/local_fs.rs index 8402daf4..52616e60 100644 --- a/objectstore-service/src/backend/local_fs.rs +++ b/objectstore-service/src/backend/local_fs.rs @@ -2,7 +2,6 @@ use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::pin::pin; -use bincode::error::DecodeError; use futures_util::StreamExt; use objectstore_types::Metadata; use tokio::fs::OpenOptions; @@ -12,8 +11,6 @@ use tokio_util::io::{ReaderStream, StreamReader}; use crate::ObjectPath; use crate::backend::common::{Backend, BackendStream}; -const BC_CONFIG: bincode::config::Configuration = bincode::config::standard(); - #[derive(Debug)] pub struct LocalFsBackend { path: PathBuf, @@ -51,8 +48,9 @@ impl Backend for LocalFsBackend { let mut reader = pin!(StreamReader::new(stream)); let mut writer = BufWriter::new(file); - let metadata = bincode::serde::encode_to_vec(metadata, BC_CONFIG)?; - writer.write_all(&metadata).await?; + let metadata_json = serde_json::to_string(metadata)?; + writer.write_all(metadata_json.as_bytes()).await?; + writer.write_all(b"\n").await?; tokio::io::copy(&mut reader, &mut writer).await?; writer.flush().await?; @@ -81,36 +79,9 @@ impl Backend for LocalFsBackend { }; let mut reader = BufReader::new(file); - let mut metadata_buf = vec![]; - // TODO populate size in our metadata - let metadata = loop { - let reader_buf = reader.fill_buf().await?; - let buf = if metadata_buf.is_empty() { - reader_buf - } else { - metadata_buf.extend_from_slice(reader_buf); - &metadata_buf - }; - - match bincode::serde::decode_from_slice(buf, BC_CONFIG) { - Ok((metadata, read)) => { - let read = if metadata_buf.is_empty() { - read - } else { - let prev_consumed = metadata_buf.len() - reader_buf.len(); - read - prev_consumed - }; - reader.consume(read); - break metadata; - } - Err(DecodeError::UnexpectedEnd { .. }) => { - metadata_buf.extend_from_slice(reader_buf); - let len = reader_buf.len(); - reader.consume(len); - } - Err(err) => Err(err)?, - } - }; + let mut metadata_line = String::new(); + reader.read_line(&mut metadata_line).await?; + let metadata: Metadata = serde_json::from_str(metadata_line.trim_end())?; let stream = ReaderStream::new(reader); Ok(Some((metadata, stream.boxed()))) diff --git a/objectstore-types/src/lib.rs b/objectstore-types/src/lib.rs index f9e6942f..a837ddb1 100644 --- a/objectstore-types/src/lib.rs +++ b/objectstore-types/src/lib.rs @@ -148,22 +148,22 @@ impl FromStr for Compression { #[serde(default)] pub struct Metadata { /// The expiration policy of the object. - // #[serde(skip_serializing_if = "ExpirationPolicy::is_manual")] + #[serde(skip_serializing_if = "ExpirationPolicy::is_manual")] pub expiration_policy: ExpirationPolicy, /// The content type of the object, if known. pub content_type: Cow<'static, str>, /// The compression algorithm used for this object, if any. - // #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "Option::is_none")] pub compression: Option, /// Size of the data in bytes, if known. - // #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "Option::is_none")] pub size: Option, /// Some arbitrary user-provided metadata. - // #[serde(skip_serializing_if = "BTreeMap::is_empty")] + #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub custom: BTreeMap, }