Skip to content

fix(oss): Use the native iobuf now published by folly #1130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ben-- wants to merge 2 commits into
base: main
Choose a base branch
from
+24 −61
Open

fix(oss): Use the native iobuf now published by folly #1130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3f64225
fix(oss): Add folly/iobuf deps for backingstore
ben-- Jun 17, 2025
93b995a
feat(oss): Remove fbcode_build guards now that iobuf is available in oss
ben-- Jun 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions eden/scm/lib/backingstore/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ blob = { version = "0.1.0", path = "../blob" }
cxx = "1.0.119"
env_logger = { version = "0.11.8", features = ["color"] }
flume = "0.11.1"
iobuf = { version = "0.1.0", git = "https://github.com/facebook/folly.git", branch = "main" }
log = { version = "0.4.27", features = ["kv_unstable", "kv_unstable_std"] }
parking_lot = { version = "0.12.1", features = ["send_guard"] }
sapling-configloader = { version = "0.1.0", path = "../config/loader" }
Expand Down
80 changes: 23 additions & 57 deletions eden/scm/lib/blob/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@

#![allow(unexpected_cfgs)]

#[cfg(fbcode_build)]
use bytes::Buf;
pub use minibytes::Bytes;
use types::Blake3;
Expand All @@ -16,7 +15,6 @@ use types::Sha1;
#[derive(Clone, Debug)]
pub enum Blob {
Bytes(minibytes::Bytes),
#[cfg(fbcode_build)]
IOBuf(iobuf::IOBufShared),
}

Expand All @@ -28,28 +26,24 @@ impl Blob {
pub fn to_bytes(&self) -> minibytes::Bytes {
match self {
Self::Bytes(bytes) => bytes.clone(),
#[cfg(fbcode_build)]
Self::IOBuf(buf) => minibytes::Bytes::from(Vec::<u8>::from(buf.clone())),
}
}

pub fn into_bytes(self) -> minibytes::Bytes {
match self {
Self::Bytes(bytes) => bytes,
#[cfg(fbcode_build)]
Self::IOBuf(buf) => minibytes::Bytes::from(Vec::<u8>::from(buf)),
}
}

pub fn into_vec(self) -> Vec<u8> {
match self {
Self::Bytes(bytes) => bytes.into(),
#[cfg(fbcode_build)]
Self::IOBuf(buf) => Vec::<u8>::from(buf),
}
}

#[cfg(fbcode_build)]
pub fn into_iobuf(self) -> iobuf::IOBufShared {
match self {
// safety: `minibytes::Bytes`'s deref as `[u8]` is valid when `bytes` is alive.
Expand All @@ -61,15 +55,13 @@ impl Blob {
pub fn len(&self) -> usize {
match self {
Self::Bytes(bytes) => bytes.len(),
#[cfg(fbcode_build)]
Self::IOBuf(buf) => buf.len(),
}
}

pub fn is_empty(&self) -> bool {
match self {
Self::Bytes(bytes) => bytes.is_empty(),
#[cfg(fbcode_build)]
Self::IOBuf(buf) => buf.is_empty(),
}
}
Expand Down Expand Up @@ -113,7 +105,6 @@ impl Blob {
) -> std::io::Result<()> {
match self {
Self::Bytes(bytes) => f(bytes),
#[cfg(fbcode_build)]
Self::IOBuf(buf) => {
let mut cur = buf.clone().cursor();
loop {
Expand All @@ -136,7 +127,6 @@ impl From<Vec<u8>> for Blob {
}
}

#[cfg(fbcode_build)]
fn iobuf_from_bytes(bytes: minibytes::Bytes) -> iobuf::IOBufShared {
unsafe { iobuf::IOBufShared::from_owner(bytes) }
}
Expand All @@ -145,13 +135,10 @@ impl PartialEq for Blob {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Self::Bytes(l), Self::Bytes(r)) => l == r,
#[cfg(fbcode_build)]
(Self::IOBuf(l), Self::IOBuf(r)) => l == r,
#[cfg(fbcode_build)]
(Self::IOBuf(buf), Self::Bytes(bytes)) => {
buf.len() == bytes.len() && buf == &iobuf_from_bytes(bytes.clone())
}
#[cfg(fbcode_build)]
(Self::Bytes(bytes), Self::IOBuf(buf)) => {
buf.len() == bytes.len() && buf == &iobuf_from_bytes(bytes.clone())
}
Expand All @@ -170,7 +157,6 @@ pub enum Builder {
// capacity
Empty(usize),
Bytes(Vec<u8>),
#[cfg(fbcode_build)]
IOBuf(iobuf::IOBufShared),
}

Expand All @@ -186,23 +172,11 @@ impl Builder {
pub fn append(&mut self, chunk: Bytes) {
match self {
Builder::Empty(size) => {
#[cfg(fbcode_build)]
{
// Using IOBuf - ignore size for pre-allocation.
let _ = size;
*self = Self::IOBuf(iobuf_from_bytes(chunk));
}

#[cfg(not(fbcode_build))]
{
// Not using IOBuf - pre-allocate with given size.
let mut data = Vec::with_capacity(*size);
data.extend_from_slice(chunk.as_ref());
*self = Self::Bytes(data);
}
// Using IOBuf - ignore size for pre-allocation.
let _ = size;
*self = Self::IOBuf(iobuf_from_bytes(chunk));
}
Builder::Bytes(data) => data.extend_from_slice(chunk.as_ref()),
#[cfg(fbcode_build)]
Builder::IOBuf(iobuf) => iobuf.append_to_end(iobuf_from_bytes(chunk)),
}
}
Expand All @@ -211,7 +185,6 @@ impl Builder {
match self {
Builder::Empty(_) => Blob::Bytes(Bytes::new()),
Builder::Bytes(data) => Blob::Bytes(data.into()),
#[cfg(fbcode_build)]
Builder::IOBuf(iobuf) => Blob::IOBuf(iobuf),
}
}
Expand All @@ -221,7 +194,6 @@ impl Builder {
mod test {
use super::*;

#[cfg(fbcode_build)]
#[test]
fn test_iobuf_sha1_and_blake3() {
let blob1 = Blob::Bytes(minibytes::Bytes::from("hello world!"));
Expand All @@ -246,18 +218,15 @@ mod test {
let b = Blob::Bytes(minibytes::Bytes::from("oops"));
assert!(a != b);

#[cfg(fbcode_build)]
{
let a = Blob::Bytes(minibytes::Bytes::from("hello world!"));
let b = Blob::IOBuf(iobuf::IOBufShared::from("hello world!"));
assert_eq!(a, b);
assert_eq!(b, a);

let a = Blob::Bytes(minibytes::Bytes::from("hello world!"));
let b = Blob::IOBuf(iobuf::IOBufShared::from("oops"));
assert!(a != b);
assert!(b != a);
}
let a = Blob::Bytes(minibytes::Bytes::from("hello world!"));
let b = Blob::IOBuf(iobuf::IOBufShared::from("hello world!"));
assert_eq!(a, b);
assert_eq!(b, a);

let a = Blob::Bytes(minibytes::Bytes::from("hello world!"));
let b = Blob::IOBuf(iobuf::IOBufShared::from("oops"));
assert!(a != b);
assert!(b != a);
}

#[test]
Expand Down Expand Up @@ -291,21 +260,18 @@ mod test {
let res = a.each_chunk(|_| Err(std::io::Error::other("oops")));
assert!(res.is_err());

#[cfg(fbcode_build)]
{
let mut iobuf = iobuf::IOBufShared::from("hello");
iobuf.append_to_end(iobuf::IOBufShared::from(""));
iobuf.append_to_end(iobuf::IOBufShared::from(" world!"));
let mut iobuf = iobuf::IOBufShared::from("hello");
iobuf.append_to_end(iobuf::IOBufShared::from(""));
iobuf.append_to_end(iobuf::IOBufShared::from(" world!"));

let a = Blob::IOBuf(iobuf);
let a = Blob::IOBuf(iobuf);

let mut got = Vec::new();
let res = a.each_chunk(|chunk| {
got.extend_from_slice(chunk);
Ok(())
});
assert_eq!(got, b"hello world!");
assert!(res.is_ok());
}
let mut got = Vec::new();
let res = a.each_chunk(|chunk| {
got.extend_from_slice(chunk);
Ok(())
});
assert_eq!(got, b"hello world!");
assert!(res.is_ok());
}
}
2 changes: 0 additions & 2 deletions eden/scm/lib/revisionstore/src/scmstore/tree.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,6 @@ impl TreeStore {
TREE_STORE_FETCH_METRICS.cas.hit(1);
let augmented_tree = match blob {
Blob::Bytes(bytes) => AugmentedTree::try_deserialize(bytes.as_ref())?,
#[allow(unexpected_cfgs)]
#[cfg(fbcode_build)]
Blob::IOBuf(buf) => AugmentedTree::try_deserialize(buf.cursor())?,
};
self.cache_child_aux_data(tree_aux_store, &augmented_tree)?;
Expand Down
2 changes: 0 additions & 2 deletions eden/scm/lib/revisionstore/src/scmstore/tree/fetch.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,8 +322,6 @@ impl FetchState {
Ok(Some(data)) => {
let deserialization_result = match data {
Blob::Bytes(bytes) => AugmentedTree::try_deserialize(bytes.as_ref()),
#[allow(unexpected_cfgs)]
#[cfg(fbcode_build)]
Blob::IOBuf(buf) => AugmentedTree::try_deserialize(buf.cursor()),
};
match deserialization_result {
Expand Down