diff --git a/Cargo.lock b/Cargo.lock index 6757ad9f..d7d5e568 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2533,6 +2533,18 @@ dependencies = [ "syn", ] +[[package]] +name = "reflink-copy" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13362233b147e57674c37b802d216b7c5e3dcccbed8967c84f0d8d223868ae27" +dependencies = [ + "cfg-if", + "libc", + "rustix", + "windows", +] + [[package]] name = "regex" version = "1.12.2" @@ -3815,6 +3827,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -3828,6 +3861,17 @@ dependencies = [ "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -3856,6 +3900,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -3967,6 +4021,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -4491,6 +4554,7 @@ dependencies = [ "indexmap 2.13.0", "num", "ouroboros", + "reflink-copy", "rkyv", "rstest", "serde", diff --git a/Cargo.toml b/Cargo.toml index 0caf8f3a..7f8e196b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ chrono = { version = "0.4.38", features = ["serde"] } crc32fast = "1.4.2" convert_case = "0.8.0" dashmap = "6" +libc = "0.2" clipanion = { git = "https://github.com/arcanis/clipanion-rs.git", features = ["serde", "tokens"] } colored = "3.0.0" dialoguer = "0.11.0" @@ -78,6 +79,7 @@ rayon = "1.10.0" rkyv = { version = "0.8", features = ["bytecheck"] } reqwest = { version = "0.12.26", default-features = false, features = ["gzip", "http2", "hickory-dns", "rustls-tls"] } regex = "1.10.6" +reflink-copy = "0.1.29" ring = "0.17.14" rstest = "0.26.1" serde_plain = "1.0.2" diff --git a/packages/zpm-utils/Cargo.toml b/packages/zpm-utils/Cargo.toml index d22fe621..cc39d7f9 100644 --- a/packages/zpm-utils/Cargo.toml +++ b/packages/zpm-utils/Cargo.toml @@ -27,3 +27,4 @@ indexmap = { workspace = true, features = ["serde"]} timeago = { workspace = true } fundu = { workspace = true } zpm-macro-enum = { workspace = true } +reflink-copy = { workspace = true } diff --git a/packages/zpm-utils/src/path.rs b/packages/zpm-utils/src/path.rs index 0dccdd1e..37abebed 100644 --- a/packages/zpm-utils/src/path.rs +++ b/packages/zpm-utils/src/path.rs @@ -769,6 +769,66 @@ impl Path { Ok(self) } + pub fn fs_clonefile(&self, new_path: &Path) -> Result<&Self, PathError> { + #[cfg(target_os = "macos")] + { + let _ = reflink_copy::reflink_or_copy(self.to_path_buf(), new_path.to_path_buf())?; + return Ok(self); + } + + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::PermissionsExt; + + fn clone_tree_linux(src: &Path, dst: &Path) -> Result<(), PathError> { + let metadata = src.fs_symlink_metadata()?; + let file_type = metadata.file_type(); + + if file_type.is_symlink() { + let target = src.fs_read_link()?; + dst.fs_create_parent()?; + dst.fs_symlink(&target)?; + return Ok(()); + } + + if file_type.is_dir() { + dst.fs_create_dir_all()?; + + for entry in src.fs_read_dir()? { + let entry = entry?; + let entry_path = Path::try_from(entry.path())?; + let entry_name = Path::try_from(entry.file_name())?; + let entry_dest = dst.with_join(&entry_name); + clone_tree_linux(&entry_path, &entry_dest)?; + } + + return Ok(()); + } + + if file_type.is_file() { + let mode = metadata.permissions().mode(); + + dst.fs_create_parent()?; + let _ = reflink_copy::reflink_or_copy(src.to_path_buf(), dst.to_path_buf())?; + dst.fs_set_permissions(std::fs::Permissions::from_mode(mode))?; + + return Ok(()); + } + + Err(std::io::Error::new(std::io::ErrorKind::Unsupported, "unsupported file type").into()) + } + + clone_tree_linux(self, new_path)?; + return Ok(self); + } + + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + let _ = new_path; + Err(std::io::Error::new(std::io::ErrorKind::Unsupported, "clonefile is only supported on macOS and Linux").into()) + } + } + /** * Move a file or directory to a new location, copying it if the source and * destination are on different devices. diff --git a/packages/zpm/src/linker/helpers.rs b/packages/zpm/src/linker/helpers.rs index a50e6997..d05a1d67 100644 --- a/packages/zpm/src/linker/helpers.rs +++ b/packages/zpm/src/linker/helpers.rs @@ -1,11 +1,13 @@ -use std::{collections::{BTreeMap, BTreeSet}, fs::Permissions, os::unix::fs::PermissionsExt, vec}; +use std::{collections::{BTreeMap, BTreeSet}, fs::Permissions, os::unix::fs::PermissionsExt, time::{SystemTime, UNIX_EPOCH}, vec}; use zpm_formats::iter_ext::IterExt; use zpm_parsers::JsonDocument; use zpm_primitives::{Descriptor, FilterDescriptor, Locator}; -use zpm_utils::{Path, PathError, System}; +use zpm_utils::{Path, PathError, System, ToFileString}; +use sha2::{Digest, Sha512}; use serde::{Deserialize, Serialize}; use serde_with::serde_as; +use hex; use crate::{ build, @@ -120,6 +122,131 @@ pub fn fs_extract_archive(destination: &Path, package_data: &PackageData) -> Res } } +// Helper function to compute SHA512 hash and return as hex string +fn compute_sha512_hex(input: &str) -> String { + let mut hasher = Sha512::new(); + hasher.update(input.as_bytes()); + hex::encode(hasher.finalize()) +} + +// Generates a Yarn Berry-compatible hash. Used for Sharp packages +pub fn yarn_berry_hash(locator: &Locator) -> Result { + let package_version = locator.reference.to_file_string(); + + // Extract scope without '@' prefix, or empty string if no scope + let package_scope = locator.ident.scope() + .and_then(|scope| scope.strip_prefix('@')) + .unwrap_or(""); + + // Step 1: Hash the package identifier (scope + name) + let package_identifier = format!("{}{}", package_scope, locator.ident.name()); + let identifier_hash = compute_sha512_hex(&package_identifier); + + // Step 2: Hash the combination of identifier hash and version + let combined_input = format!("{}{}", identifier_hash, package_version); + let final_hash = compute_sha512_hex(&combined_input); + + // Return first 10 characters to match Yarn Berry's hash length + Ok(final_hash[..10].to_string()) +} + +pub fn fs_materialize_unplugged_from_global_cache(project: &Project, locator: &Locator, dest_wrapper: &Path, dest_package_root: &Path, package_data: &PackageData) -> Result { + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + let _ = (project, locator, dest_wrapper); + return fs_extract_archive(dest_package_root, package_data); + } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + let dest_ready = dest_package_root + .with_join_str(".ready"); + + if dest_ready.fs_exists() { + return Ok(false); + } + + let PackageData::Zip {..} = package_data else { + return Ok(false); + }; + + let global_base = project + .global_unplugged_path(); + + global_base + .fs_create_dir_all()?; + + let physical = locator + .physical_locator(); + + let global_wrapper_name = format!( + "{}-{}-{}", + physical.ident.slug(), + physical.reference.slug(), + yarn_berry_hash(&physical)?, + ); + + let global_wrapper = global_base + .with_join_str(&global_wrapper_name); + + let package_subpath = package_data + .package_subpath(); + + let global_package_root = global_wrapper + .with_join(&package_subpath); + + let global_ready = global_package_root + .with_join_str(".ready"); + + if !global_ready.fs_exists() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + + let tmp_wrapper = global_base.with_join_str(format!( + ".{}.tmp-{}-{}", + global_wrapper_name, + std::process::id(), + nonce, + )); + + let tmp_package_root = tmp_wrapper + .with_join(&package_subpath); + + if fs_extract_archive(&tmp_package_root, package_data).is_ok() { + let _ = tmp_wrapper + .fs_concurrent_move(&global_wrapper); + } + + if !global_ready.fs_exists() { + let _ = tmp_wrapper.fs_rm(); + return fs_extract_archive(dest_package_root, package_data); + } + + let _ = tmp_wrapper.fs_rm(); + } + + if dest_wrapper.fs_exists() && !dest_ready.fs_exists() { + dest_wrapper.fs_rm()?; + } + + dest_wrapper + .fs_create_parent()?; + + match global_wrapper.fs_clonefile(dest_wrapper) { + Ok(_) => Ok(true), + Err(_) => { + if dest_wrapper.fs_exists() { + let _ = dest_wrapper.fs_rm(); + } + + fs_extract_archive(dest_package_root, package_data) + }, + } + } +} + pub fn populate_build_entry_dependencies(package_build_entries: &BTreeMap, locator_resolutions: &BTreeMap, descriptor_to_locator: &BTreeMap) -> Result>, Error> { let mut package_build_dependencies = BTreeMap::new(); diff --git a/packages/zpm/src/linker/pnp.rs b/packages/zpm/src/linker/pnp.rs index 8e2a3e1e..c6ead3df 100644 --- a/packages/zpm/src/linker/pnp.rs +++ b/packages/zpm/src/linker/pnp.rs @@ -4,8 +4,6 @@ use zpm_config::PnpFallbackMode; use zpm_parsers::JsonDocument; use zpm_primitives::{Ident, Locator, Reference}; use zpm_utils::{IoResultExt, Path, SyncEntryKind, ToHumanString}; -use sha2::{Sha512, Digest}; -use hex; use itertools::Itertools; use serde::{Serialize, Serializer}; use serde_with::serde_as; @@ -52,34 +50,6 @@ fn make_virtual_path(base: &Path, component: &str, to: &Path) -> Path { full_virtual_path } -// Helper function to compute SHA512 hash and return as hex string -fn compute_sha512_hex(input: &str) -> String { - let mut hasher = Sha512::new(); - hasher.update(input.as_bytes()); - hex::encode(hasher.finalize()) -} - -// Generates a Yarn Berry-compatible hash. Used for Sharp packages -fn yarn_berry_hash(locator: &Locator) -> Result { - let package_version = locator.reference.to_file_string(); - - // Extract scope without '@' prefix, or empty string if no scope - let package_scope = locator.ident.scope() - .and_then(|scope| scope.strip_prefix('@')) - .unwrap_or(""); - - // Step 1: Hash the package identifier (scope + name) - let package_identifier = format!("{}{}", package_scope, locator.ident.name()); - let identifier_hash = compute_sha512_hex(&package_identifier); - - // Step 2: Hash the combination of identifier hash and version - let combined_input = format!("{}{}", identifier_hash, package_version); - let final_hash = compute_sha512_hex(&combined_input); - - // Return first 10 characters to match Yarn Berry's hash length - Ok(final_hash[..10].to_string()) -} - #[serde_as] #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] struct PnpReference(Locator); @@ -334,7 +304,7 @@ pub async fn link_project_pnp<'a>(project: &'a Project, install: &'a Install) -> is_physically_on_disk = true; } else if package_build_info.must_extract { let package_unplugged_wrapper_path = unplugged_path - .with_join_str(format!("{}-{}-{}", locator.ident.slug(), locator.reference.slug(), yarn_berry_hash(locator)?)); + .with_join_str(format!("{}-{}-{}", locator.ident.slug(), locator.reference.slug(), linker::helpers::yarn_berry_hash(locator)?)); package_location_abs = package_unplugged_wrapper_path .with_join(&physical_package_data.package_subpath()); @@ -347,7 +317,10 @@ pub async fn link_project_pnp<'a>(project: &'a Project, install: &'a Install) -> package_location_abs.clone(), ); - is_freshly_unplugged = linker::helpers::fs_extract_archive( + is_freshly_unplugged = linker::helpers::fs_materialize_unplugged_from_global_cache( + project, + locator, + &package_unplugged_wrapper_path, &package_location_abs, physical_package_data, )?; diff --git a/packages/zpm/src/project.rs b/packages/zpm/src/project.rs index fbbb0d60..5c8b2cb6 100644 --- a/packages/zpm/src/project.rs +++ b/packages/zpm/src/project.rs @@ -253,6 +253,11 @@ impl Project { .with_join_str("cache") } + pub fn global_unplugged_path(&self) -> Path { + self.config.settings.global_folder.value + .with_join_str("unplugged") + } + pub fn local_cache_path(&self) -> Path { self.project_cwd .with_join_str(".yarn")