diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index d5948b0f..a069d3cd 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -11,10 +11,11 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, macos-latest, windows-2019] + os: [ubuntu-22.04, macos-latest, windows-2019] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - uses: taiki-e/install-action@cargo-hack - uses: arduino/setup-protoc@v2 with: version: "23.1" @@ -30,6 +31,7 @@ jobs: - name: Env run: | env|sort + xcodebuild -version - name: Build and run tests on windows if: runner.os == 'Windows' shell: pwsh @@ -90,51 +92,65 @@ jobs: popd done + - - name: Build - if: runner.os != 'Windows' - shell: bash + - name: Build and Test (Linux) + if: runner.os == 'Linux' + shell: bash -ex {0} run: | - set +x export RUST_BACKTRACE=1 - - cd rust - if [ $RUNNER_OS = 'macOS' ]; then - brew install openssl@3 - # Upgrading make is required for cargo to play nicely with building openssl - brew install make - export PATH="/usr/local/opt/make/libexec/gnubin:$PATH" - fi - - cargo build --verbose --features "strict openssl_vendored" - - name: Test Environment - if: runner.os != 'Windows' - shell: bash - run: | - set +x - if [ $RUNNER_OS = 'macOS' ]; then - brew install md5sha1sum - fi - cd rust # we need a basic git configuration for the tests to pass git config --global user.email operations@xetdata.com git config --global user.name "XetData Automation" - - name: Test - if: runner.os != 'Windows' - shell: bash + + pushd rust + cargo test --verbose --no-fail-fast --features "strict openssl_vendored" + popd + + pushd gitxet + cargo test --verbose --no-fail-fast --features "openssl_vendored" + popd + + pushd xetldfs + cargo test --verbose --no-fail-fast --features "openssl_vendored" + popd + + - name: Setup (macOS) + if: runner.os == 'macOS' + shell: bash -ex {0} run: | - set +x - export RUST_BACKTRACE=1 + brew install openssl@3 + + # Upgrading make is required for cargo to play nicely with building openssl + brew install make + export PATH="/usr/local/opt/make/libexec/gnubin:$PATH" + + # Use our own shell to avoid arm64 vs arm64e issues + brew install bash - cd rust - cargo test --no-fail-fast --features "strict openssl_vendored" - - name: Integration tests - if: runner.os != 'Windows' - shell: bash + brew install md5sha1sum + + - name: Build and Test (macOS) + if: runner.os == 'macOS' + shell: /opt/homebrew/bin/bash -ex {0} run: | - set +x - export RUST_BACKTRACE=1 + export PATH="/opt/homebrew/bin/:/usr/local/opt/make/libexec/gnubin:$PATH" + + # we need a basic git configuration for the tests to pass + git config --global user.email operations@xetdata.com + git config --global user.name "XetData Automation" + + pushd rust + cargo test --verbose --no-fail-fast --features "strict openssl_vendored" + popd + + pushd gitxet + cargo test --verbose --no-fail-fast --features "openssl_vendored" + popd - cd gitxet - cargo test --no-fail-fast --features "openssl_vendored" + # This is dumb. + pushd xetldfs + cargo test --verbose --no-fail-fast --features "openssl_vendored" --target=x86_64-apple-darwin + popd + diff --git a/gitxet/Cargo.lock b/gitxet/Cargo.lock index c59ba3f2..e1bf86d9 100644 --- a/gitxet/Cargo.lock +++ b/gitxet/Cargo.lock @@ -50,6 +50,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -153,18 +159,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -178,12 +184,9 @@ dependencies = [ [[package]] name = "atomic" -version = "0.6.0" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" -dependencies = [ - "bytemuck", -] +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" [[package]] name = "atomic-waker" @@ -312,9 +315,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "blake3" @@ -444,7 +447,7 @@ dependencies = [ "futures", "http 0.2.12", "http-body-util", - "hyper 1.4.0", + "hyper 1.3.1", "hyper-rustls", "hyper-util", "itertools 0.10.5", @@ -477,9 +480,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.106" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "066fce287b1d4eafef758e89e09d724a24808a9196fe9756b8ca90e86d0719a2" +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" dependencies = [ "jobserver", "libc", @@ -516,7 +519,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.6", + "windows-targets 0.52.5", ] [[package]] @@ -828,7 +831,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -845,7 +848,7 @@ checksum = "4b2c1c1776b986979be68bb2285da855f8d8a35851a769fca8740df7c3d07877" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -1017,9 +1020,9 @@ checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" [[package]] name = "either" -version = "1.13.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" [[package]] name = "encoding_rs" @@ -1039,7 +1042,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -1237,7 +1240,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -1346,7 +1349,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -1354,7 +1357,7 @@ name = "git2" version = "0.18.2" source = "git+https://github.com/xetdata/git2-rs#dfffd3d1eb9e87a9e7e98b24d0a0f54db664cbcc" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.5.0", "libc", "libgit2-sys", "log", @@ -1463,6 +1466,7 @@ dependencies = [ "slog-json", "snailquote", "sorted-vec", + "static_assertions", "sysinfo", "tableau_summary", "tabled", @@ -1546,6 +1550,10 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash 0.8.11", + "allocator-api2", +] [[package]] name = "hashers" @@ -1746,9 +1754,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ "bytes", "futures-channel", @@ -1772,11 +1780,11 @@ checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.0", + "hyper 1.3.1", "hyper-util", "log", "rustls 0.22.4", - "rustls-native-certs 0.7.1", + "rustls-native-certs 0.7.0", "rustls-pki-types", "tokio", "tokio-rustls 0.25.0", @@ -1810,16 +1818,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.4.0", + "hyper 1.3.1", "pin-project-lite", "socket2", "tokio", @@ -1869,12 +1877,12 @@ dependencies = [ [[package]] name = "imara-diff" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af13c8ceb376860ff0c6a66d83a8cdd4ecd9e464da24621bbffcd02b49619434" +checksum = "e98c1d0ad70fc91b8b9654b1f33db55e59579d3b3de2bffdced0fdb810570cb8" dependencies = [ "ahash 0.8.11", - "hashbrown 0.14.5", + "hashbrown 0.12.3", ] [[package]] @@ -2013,9 +2021,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "len-trait" @@ -2063,7 +2071,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.5.0", "libc", ] @@ -2071,7 +2079,9 @@ dependencies = [ name = "libxet" version = "0.14.5" dependencies = [ + "error_printer", "gitxetcore", + "merkledb", "progress_reporting", ] @@ -2131,17 +2141,17 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "lru" -version = "0.7.8" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ - "hashbrown 0.12.3", + "hashbrown 0.14.5", ] [[package]] @@ -2324,7 +2334,7 @@ dependencies = [ "cfg-if 1.0.0", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -2388,7 +2398,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.5.0", "cfg-if 1.0.0", "cfg_aliases", "libc", @@ -2498,7 +2508,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.5.0", "cfg-if 1.0.0", "foreign-types", "libc", @@ -2515,7 +2525,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -2706,7 +2716,7 @@ dependencies = [ "libc", "redox_syscall 0.5.2", "smallvec", - "windows-targets 0.52.6", + "windows-targets 0.52.5", ] [[package]] @@ -2767,9 +2777,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.11" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" dependencies = [ "memchr", "thiserror", @@ -2778,9 +2788,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.11" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" dependencies = [ "pest", "pest_generator", @@ -2788,22 +2798,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.11" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] name = "pest_meta" -version = "2.7.11" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" dependencies = [ "once_cell", "pest", @@ -2850,7 +2860,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -2879,7 +2889,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -2960,7 +2970,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -3059,7 +3069,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.69", + "syn 2.0.67", "tempfile", ] @@ -3073,7 +3083,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -3211,7 +3221,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.5.0", ] [[package]] @@ -3449,7 +3459,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", @@ -3477,7 +3487,7 @@ dependencies = [ "log", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.5", + "rustls-webpki 0.102.4", "subtle", "zeroize", ] @@ -3496,9 +3506,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", "rustls-pemfile 2.1.2", @@ -3544,9 +3554,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.5" +version = "0.102.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -3617,7 +3627,7 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.5.0", "core-foundation", "core-foundation-sys", "libc", @@ -3642,29 +3652,29 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "itoa", "ryu", @@ -3902,6 +3912,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.8.0" @@ -3959,9 +3975,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.6.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +checksum = "0d0208408ba0c3df17ed26eb06992cb1a1268d41b2c0e12e65203fbe3972cee5" [[package]] name = "syn" @@ -3976,9 +3992,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.69" +version = "2.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201fcda3845c23e8212cd466bfebf0bd20694490fc0356ae8e428e0824a915a6" +checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" dependencies = [ "proc-macro2", "quote", @@ -4152,7 +4168,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -4220,9 +4236,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.7.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6b6a2fb3a985e99cebfaefa9faa3024743da73304ca1c683a36429613d3d22" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -4270,7 +4286,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -4402,7 +4418,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -4457,7 +4473,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -4575,7 +4591,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -4710,9 +4726,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.9.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ "atomic", "getrandom", @@ -4810,7 +4826,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", "wasm-bindgen-shared", ] @@ -4844,7 +4860,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4919,7 +4935,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.6", + "windows-targets 0.52.5", ] [[package]] @@ -4937,7 +4953,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets 0.52.5", ] [[package]] @@ -4957,18 +4973,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -4979,9 +4995,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -4991,9 +5007,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -5003,15 +5019,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" [[package]] name = "windows_i686_gnullvm" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -5021,9 +5037,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -5033,9 +5049,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -5045,9 +5061,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -5057,9 +5073,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.6" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winreg" @@ -5077,7 +5093,7 @@ version = "1.0.50" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] @@ -5111,22 +5127,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.35" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.35" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.67", ] [[package]] diff --git a/libxet/Cargo.lock b/libxet/Cargo.lock index 7a74f91c..26f28ce7 100644 --- a/libxet/Cargo.lock +++ b/libxet/Cargo.lock @@ -50,6 +50,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -132,18 +138,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -200,7 +206,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "itoa", "matchit", "memchr", @@ -412,7 +418,7 @@ dependencies = [ "futures", "http 0.2.12", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-rustls", "hyper-util", "itertools 0.10.5", @@ -445,9 +451,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.104" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" +checksum = "eaff6f8ce506b9773fa786672d63fc7a191ffea1be33f72bbd4aeacefca9ffc8" dependencies = [ "jobserver", "libc", @@ -484,7 +490,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -796,7 +802,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -813,7 +819,7 @@ checksum = "4b2c1c1776b986979be68bb2285da855f8d8a35851a769fca8740df7c3d07877" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -995,7 +1001,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1043,6 +1049,21 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +[[package]] +name = "file_utils" +version = "0.14.2" +dependencies = [ + "anyhow", + "colored", + "lazy_static", + "libc", + "rand 0.8.5", + "tempfile", + "tracing", + "whoami", + "winapi", +] + [[package]] name = "filetime" version = "0.2.23" @@ -1168,7 +1189,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1277,7 +1298,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1319,6 +1340,7 @@ dependencies = [ "enum_dispatch", "error_printer", "fallible-iterator", + "file_utils", "filetime", "futures", "futures-core", @@ -1374,6 +1396,7 @@ dependencies = [ "slog-json", "snailquote", "sorted-vec", + "static_assertions", "sysinfo", "tableau_summary", "tabled", @@ -1457,6 +1480,10 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash 0.8.11", + "allocator-api2", +] [[package]] name = "hashers" @@ -1633,9 +1660,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.29" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", @@ -1657,9 +1684,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", @@ -1683,11 +1710,11 @@ checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "log", "rustls 0.22.4", - "rustls-native-certs 0.7.0", + "rustls-native-certs 0.7.1", "rustls-pki-types", "tokio", "tokio-rustls 0.25.0", @@ -1700,7 +1727,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.29", + "hyper 0.14.30", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -1713,7 +1740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.29", + "hyper 0.14.30", "native-tls", "tokio", "tokio-native-tls", @@ -1730,7 +1757,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.4.0", + "hyper 1.4.1", "pin-project-lite", "socket2", "tokio", @@ -1780,12 +1807,12 @@ dependencies = [ [[package]] name = "imara-diff" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e98c1d0ad70fc91b8b9654b1f33db55e59579d3b3de2bffdced0fdb810570cb8" +checksum = "af13c8ceb376860ff0c6a66d83a8cdd4ecd9e464da24621bbffcd02b49619434" dependencies = [ "ahash 0.8.11", - "hashbrown 0.12.3", + "hashbrown 0.14.5", ] [[package]] @@ -1982,7 +2009,9 @@ dependencies = [ name = "libxet" version = "0.14.5" dependencies = [ + "error_printer", "gitxetcore", + "merkledb", "progress_reporting", ] @@ -2048,11 +2077,11 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.7.8" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ - "hashbrown 0.12.3", + "hashbrown 0.14.5", ] [[package]] @@ -2235,7 +2264,7 @@ dependencies = [ "cfg-if 1.0.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2420,7 +2449,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2611,7 +2640,7 @@ dependencies = [ "libc", "redox_syscall 0.5.2", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2672,9 +2701,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" +checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" dependencies = [ "memchr", "thiserror", @@ -2683,9 +2712,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" +checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" dependencies = [ "pest", "pest_generator", @@ -2693,22 +2722,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" +checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] name = "pest_meta" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" +checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" dependencies = [ "once_cell", "pest", @@ -2755,7 +2784,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2784,7 +2813,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2854,7 +2883,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2953,7 +2982,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.68", + "syn 2.0.70", "tempfile", ] @@ -2967,7 +2996,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3186,7 +3215,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-tls", "ipnet", "js-sys", @@ -3340,7 +3369,7 @@ dependencies = [ "log", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.4", + "rustls-webpki 0.102.5", "subtle", "zeroize", ] @@ -3359,9 +3388,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ "openssl-probe", "rustls-pemfile 2.1.2", @@ -3407,9 +3436,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -3499,22 +3528,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3597,7 +3626,7 @@ dependencies = [ "clap 2.34.0", "heed", "http 0.2.12", - "hyper 0.14.29", + "hyper 0.14.30", "itertools 0.10.5", "lazy_static", "mdb_shard", @@ -3759,6 +3788,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.8.0" @@ -3833,9 +3868,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.68" +version = "2.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" dependencies = [ "proc-macro2", "quote", @@ -4009,7 +4044,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4077,9 +4112,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -4127,7 +4162,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4219,7 +4254,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-timeout", "percent-encoding", "pin-project", @@ -4246,7 +4281,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4301,7 +4336,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4419,7 +4454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4554,9 +4589,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "atomic", "getrandom", @@ -4645,7 +4680,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", "wasm-bindgen-shared", ] @@ -4679,7 +4714,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4754,7 +4789,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -4772,7 +4807,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -4792,18 +4827,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -4814,9 +4849,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -4826,9 +4861,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -4838,15 +4873,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -4856,9 +4891,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -4868,9 +4903,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -4880,9 +4915,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -4892,9 +4927,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winreg" @@ -4912,7 +4947,7 @@ version = "1.0.50" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4946,22 +4981,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] diff --git a/libxet/Cargo.toml b/libxet/Cargo.toml index f5559e8d..ad57498b 100644 --- a/libxet/Cargo.toml +++ b/libxet/Cargo.toml @@ -5,7 +5,9 @@ edition = "2021" [dependencies] gitxetcore = { path = "../rust/gitxetcore" } +merkledb = { path = "../rust/merkledb" } progress_reporting = { path = "../rust/progress_reporting" } +error_printer = { path = "../rust/error_printer" } [features] openssl_vendored = ["gitxetcore/openssl_vendored"] @@ -25,4 +27,3 @@ debug = "full" inherits = "dev" opt-level = 1 debug = 1 - diff --git a/libxet/src/lib.rs b/libxet/src/lib.rs index 970884b3..854a03c3 100644 --- a/libxet/src/lib.rs +++ b/libxet/src/lib.rs @@ -6,5 +6,7 @@ // Add more exports as needed here. // +pub use error_printer::ErrorPrinter; pub use gitxetcore::*; +pub use merkledb; pub use progress_reporting; diff --git a/rust/cache/Cargo.toml b/rust/cache/Cargo.toml index 5438000f..9bed6179 100644 --- a/rust/cache/Cargo.toml +++ b/rust/cache/Cargo.toml @@ -6,18 +6,18 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -utils = {path = "../utils"} +utils = { path = "../utils" } tokio = { version = "1.36", features = ["full"] } async-trait = "0.1.9" base64 = "0.13.0" -lru = "0.7.3" -xet_error = {path = "../xet_error"} +lru = "0.12" +xet_error = { path = "../xet_error" } anyhow = "1" tracing = "0.1.31" tracing-attributes = "0.1" tracing-futures = "0.2" tracing-test = "0.2.1" -tracing-subscriber = {version = "0.3", features = ["json", "tracing-log"]} +tracing-subscriber = { version = "0.3", features = ["json", "tracing-log"] } mockall = "0.11" chrono = "0.4" tempfile = "3.2.0" @@ -30,7 +30,7 @@ prometheus = "0.13.0" tempdir = "0.3.7" rand = "0.8.5" test-context = "0.1.3" -merklehash = { path = "../merklehash"} +merklehash = { path = "../merklehash" } [features] strict = [] diff --git a/rust/cache/src/lru.rs b/rust/cache/src/lru.rs index 17d74bcd..de46d6eb 100644 --- a/rust/cache/src/lru.rs +++ b/rust/cache/src/lru.rs @@ -35,7 +35,7 @@ impl Default for Lru { impl Lru { pub fn new(capacity: usize, duration_minutes: i64, name: &str) -> Self { Self { - lru: LruCache::new(capacity), + lru: LruCache::new(std::num::NonZero::new(capacity.max(1)).unwrap()), duration: { #[allow(deprecated)] // Sometimes the linter seems to hate this.... Duration::minutes(duration_minutes) diff --git a/rust/gitxetcore/Cargo.toml b/rust/gitxetcore/Cargo.toml index f516ae63..5f215ea7 100644 --- a/rust/gitxetcore/Cargo.toml +++ b/rust/gitxetcore/Cargo.toml @@ -70,7 +70,7 @@ libmagic = { path = "../libmagic" } sorted-vec = "0.8.0" bincode = "1.3.3" enum_dispatch = "0.3.8" -lru = "0.7.8" +lru = "0.12" intaglio = "1.8.0, <1.9.0" walkdir = "2" filetime = "0.2" @@ -102,6 +102,7 @@ uuid = { version = "1.8.0", features = ["std", "rng", "v6"] } lz4 = "1.24.0" git-url-parse = "0.4.4" path-absolutize = "3.1.1" # Can drop after rust 1.79 +static_assertions = "1.1.0" # tracing tracing-futures = "0.2" diff --git a/rust/gitxetcore/src/command/materialize.rs b/rust/gitxetcore/src/command/materialize.rs index 2098e42f..59de6371 100644 --- a/rust/gitxetcore/src/command/materialize.rs +++ b/rust/gitxetcore/src/command/materialize.rs @@ -1,15 +1,13 @@ -use crate::data::PointerFile; +use crate::data::smudge_pointerfile_to_itself; use clap::Args; use itertools::Itertools; use lazy::lazy_pathlist_config::{check_or_create_lazy_config, LazyPathListConfigFile}; use parutils::tokio_par_for_each; -use std::fs::File; -use std::io::BufWriter; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::Arc; use tracing::error; -use crate::constants::{MAX_CONCURRENT_DOWNLOADS, POINTER_FILE_LIMIT}; +use crate::constants::MAX_CONCURRENT_DOWNLOADS; use crate::data::PointerFileTranslator; use crate::errors::Result; use crate::git_integration::{filter_files_from_index, walk_working_dir, GitXetRepo}; @@ -88,7 +86,7 @@ pub async fn materialize_command(cfg: XetConfig, args: &MaterializeArgs) -> Resu *MAX_CONCURRENT_DOWNLOADS, |path, _| async move { let translator = translator_ref.clone(); - smudge_file_to_itself(&translator, &path).await + smudge_pointerfile_to_itself(&translator, &path).await }, ) .await @@ -106,33 +104,3 @@ pub async fn materialize_command(cfg: XetConfig, args: &MaterializeArgs) -> Resu Ok(()) } - -/// Smudge a pointer file and overwrite itself -async fn smudge_file_to_itself( - translator: &PointerFileTranslator, - path: &Path, -) -> anyhow::Result<()> { - let size = std::fs::metadata(path)?.len(); - - // quick check if likely a pointer file - if size > POINTER_FILE_LIMIT as u64 { - return Ok(()); - } - - let pointer_file = PointerFile::init_from_path(path.to_str().unwrap_or_default()); - - // not a pointer file, leave it as it is. - if !pointer_file.is_valid() { - return Ok(()); - } - - let file_hash = pointer_file.hash()?; - - let mut writer = Box::new(BufWriter::new(File::create(path)?)); - - translator - .smudge_file_from_hash(Some(path.to_owned()), &file_hash, &mut writer, None) - .await?; - - Ok(()) -} diff --git a/rust/gitxetcore/src/command/summary.rs b/rust/gitxetcore/src/command/summary.rs index cb8dbf5e..89e073b0 100644 --- a/rust/gitxetcore/src/command/summary.rs +++ b/rust/gitxetcore/src/command/summary.rs @@ -1,11 +1,7 @@ -use std::io::stdin; -use std::{ - fs, - path::{Path, PathBuf}, -}; - use clap::{Args, Subcommand}; use serde::Serialize; +use std::io::stdin; +use std::path::{Path, PathBuf}; use tracing::warn; use chunkpipe::pipe; @@ -156,15 +152,10 @@ async fn print_summary( summary_type: &SummaryType, file_path: &Path, ) -> errors::Result<()> { - // first, see if the input file is a pointer file - let size = fs::metadata(file_path)?.len(); - if size <= POINTER_FILE_LIMIT as u64 { - // see if it's a pointer - let file_path_str = file_path.to_string_lossy().to_string(); - let pointer_file = PointerFile::init_from_path(&file_path_str); - if pointer_file.is_valid() { - return print_summary_from_db(config, &pointer_file, summary_type).await; - } + // see if it's a pointer + let pointer_file = PointerFile::init_from_path(file_path); + if pointer_file.is_valid() { + return print_summary_from_db(config, &pointer_file, summary_type).await; } // fall through. Non-pointer. match summary_type { diff --git a/rust/gitxetcore/src/config/authentication.rs b/rust/gitxetcore/src/config/authentication.rs index df25390d..8f28058b 100644 --- a/rust/gitxetcore/src/config/authentication.rs +++ b/rust/gitxetcore/src/config/authentication.rs @@ -7,11 +7,13 @@ use std::{ }; use tracing::info; +use crate::environment::CommandSanitized; + #[derive(Default)] pub struct XeteaAuth {} fn exec_git_credential(command: &str) -> Result { - Ok(Command::new("git") + Ok(Command::new_sanitized("git") .args(["credential", command]) .env("GCM_INTERACTIVE", "never") .env("GIT_TERMINAL_PROMPT", "0") diff --git a/rust/gitxetcore/src/config/git_path.rs b/rust/gitxetcore/src/config/git_path.rs index 82a4a1f7..5884ba2e 100644 --- a/rust/gitxetcore/src/config/git_path.rs +++ b/rust/gitxetcore/src/config/git_path.rs @@ -104,8 +104,8 @@ mod test_git_config_path { fn test_discover_into_git_path() { let tmp_repo = git_repo_test_tools::TestRepoPath::new("git-path-discover").unwrap(); let path = tmp_repo.path; - let expected_path = path.join(".git"); run_git_captured(Some(&path), "init", &[], true, None).unwrap(); + let expected_path = path.join(".git").canonicalize().unwrap(); // need canonical path since MacOS symlinks /var -> /private/var let cfg_option = ConfigGitPathOption::PathDiscover(path); let git_path = cfg_option.into_git_path().unwrap(); assert_eq!(expected_path, git_path); @@ -155,7 +155,8 @@ mod test_git_config_path { let tmp_repo = git_repo_test_tools::TestRepoPath::new("into_repo_info").unwrap(); let path = tmp_repo.path; run_git_captured(Some(&path), "init", &[], true, None).unwrap(); - let expected_path = path.join(".git"); + let expected_path = path.join(".git").canonicalize().unwrap(); // need canonical path since MacOS symlinks /var -> /private/var + // Add a remote repo let repo_url = "https://xethub.com/org/repo"; add_remote_repo(&path, "origin", repo_url); @@ -172,7 +173,8 @@ mod test_git_config_path { let tmp_repo = git_repo_test_tools::TestRepoPath::new("into_repo_info_multi").unwrap(); let path = tmp_repo.path; run_git_captured(Some(&path), "init", &[], true, None).unwrap(); - let expected_path = path.join(".git"); + let expected_path = path.join(".git").canonicalize().unwrap(); // need canonical path since MacOS symlinks /var -> /private/var + // Add a couple remote repos let origin_url = "https://xethub.com/org/repo"; add_remote_repo(&path, "origin", origin_url); @@ -191,7 +193,8 @@ mod test_git_config_path { let tmp_repo = git_repo_test_tools::TestRepoPath::new("into_repo_info_multi_same").unwrap(); let path = tmp_repo.path; run_git_captured(Some(&path), "init", &[], true, None).unwrap(); - let expected_path = path.join(".git"); + let expected_path = path.join(".git").canonicalize().unwrap(); // need canonical path since MacOS symlinks /var -> /private/var + // Add a couple remote repos let origin_url = "https://xethub.com/org/repo"; add_remote_repo(&path, "origin", origin_url); diff --git a/rust/gitxetcore/src/constants.rs b/rust/gitxetcore/src/constants.rs index b3d33556..dd8c901c 100644 --- a/rust/gitxetcore/src/constants.rs +++ b/rust/gitxetcore/src/constants.rs @@ -34,10 +34,9 @@ pub const GIT_MAX_PACKET_SIZE: usize = 65516; /// We put a limit on the pointer file size so that /// we don't ever try to read a whole giant blob into memory when -/// trying to clean. The maximum git packet size is 65516. -/// By setting this threshold to 65515, we can ensure that reading exactly -/// 1 packet is enough to determine if it is a valid pointer file. -pub const POINTER_FILE_LIMIT: usize = GIT_MAX_PACKET_SIZE - 1; +/// trying to clean or smudge. +/// See gitxetcore::data::pointer_file for the explanation for this limit. +pub const POINTER_FILE_LIMIT: usize = 150; /// If a file has size smaller than this threshold, AND if it "looks-like" /// text, we interpret this as a text file and passthrough the file, letting diff --git a/rust/gitxetcore/src/data/data_processing_v1.rs b/rust/gitxetcore/src/data/data_processing_v1.rs index 2ec41e6c..095a7426 100644 --- a/rust/gitxetcore/src/data/data_processing_v1.rs +++ b/rust/gitxetcore/src/data/data_processing_v1.rs @@ -130,8 +130,12 @@ impl PointerFileTranslatorV1 { small_file_threshold: config.cas.size_threshold, cas_accumulator: Arc::new(Mutex::new(CasAccumulator::default())), prefetches: Arc::new(Mutex::new(HashMap::new())), - prefetched: Arc::new(Mutex::new(LruCache::new(PREFETCH_TRACK_COUNT))), - derive_blocks_cache: Mutex::new(LruCache::new(DERIVE_BLOCKS_CACHE_COUNT)), + prefetched: Arc::new(Mutex::new(LruCache::new( + std::num::NonZero::new(PREFETCH_TRACK_COUNT).unwrap(), + ))), + derive_blocks_cache: Mutex::new(LruCache::new( + std::num::NonZero::new(DERIVE_BLOCKS_CACHE_COUNT).unwrap(), + )), cfg: config.clone(), lazyconfig, }) @@ -152,8 +156,12 @@ impl PointerFileTranslatorV1 { small_file_threshold: config.cas.size_threshold, cas_accumulator: Arc::new(Mutex::new(CasAccumulator::default())), prefetches: Arc::new(Mutex::new(HashMap::new())), - prefetched: Arc::new(Mutex::new(LruCache::new(PREFETCH_TRACK_COUNT))), - derive_blocks_cache: Mutex::new(LruCache::new(DERIVE_BLOCKS_CACHE_COUNT)), + prefetched: Arc::new(Mutex::new(LruCache::new( + std::num::NonZero::new(PREFETCH_TRACK_COUNT).unwrap(), + ))), + derive_blocks_cache: Mutex::new(LruCache::new( + std::num::NonZero::new(DERIVE_BLOCKS_CACHE_COUNT).unwrap(), + )), cfg: config.clone(), lazyconfig: None, }) @@ -830,8 +838,12 @@ impl PointerFileTranslatorV1 { small_file_threshold: crate::constants::SMALL_FILE_THRESHOLD, cas_accumulator: Arc::new(Mutex::new(CasAccumulator::default())), prefetches: Arc::new(Mutex::new(HashMap::new())), - prefetched: Arc::new(Mutex::new(LruCache::new(PREFETCH_TRACK_COUNT))), - derive_blocks_cache: Mutex::new(LruCache::new(DERIVE_BLOCKS_CACHE_COUNT)), + prefetched: Arc::new(Mutex::new(LruCache::new( + std::num::NonZero::new(PREFETCH_TRACK_COUNT).unwrap(), + ))), + derive_blocks_cache: Mutex::new(LruCache::new( + std::num::NonZero::new(DERIVE_BLOCKS_CACHE_COUNT).unwrap(), + )), cfg: XetConfig::default(), lazyconfig: None, } diff --git a/rust/gitxetcore/src/data/pointer_file.rs b/rust/gitxetcore/src/data/pointer_file.rs index 572ea478..a3c2f5d1 100644 --- a/rust/gitxetcore/src/data/pointer_file.rs +++ b/rust/gitxetcore/src/data/pointer_file.rs @@ -1,12 +1,17 @@ #![cfg_attr(feature = "strict", deny(warnings))] - -use std::{collections::BTreeMap, fs, path::Path}; - -use crate::errors::Result; use merklehash::{DataHashHexParseError, MerkleHash}; +use static_assertions::const_assert; +use std::{ + collections::BTreeMap, + fs::{self, File}, + io::BufWriter, + path::Path, +}; use toml::Value; -use tracing::{error, warn}; +use tracing::{debug, error, warn}; +use super::PointerFileTranslator; +use crate::errors::Result; use crate::{constants::POINTER_FILE_LIMIT, stream::data_iterators::AsyncDataIterator}; const HEADER_PREFIX: &str = "# xet version "; @@ -123,19 +128,37 @@ impl PointerFile { } } - pub fn init_from_path(path: &str) -> PointerFile { + /// Initialize a pointer file by the contents in the file. + /// This will quickly check the file size before trying to read the + /// entire file. Any I/O failure or file size exceeding a limit means + /// an invalid pointer file. + pub fn init_from_path(path: impl AsRef) -> PointerFile { + let path = path.as_ref().to_str().unwrap(); let empty_string = "".to_string(); - let contents = match fs::read_to_string(path) { - Ok(s) => s, - Err(_) => { - return PointerFile { - version_string: empty_string.clone(), - path: path.to_string(), - is_valid: false, - hash: empty_string, - filesize: 0, - } - } + + let invalid_pointer_file = || PointerFile { + version_string: empty_string.clone(), + path: path.to_owned(), + is_valid: false, + hash: empty_string, + filesize: 0, + }; + + let Ok(file_meta) = fs::metadata(path).map_err(|e| { + debug!("fs:metadata failed: {e:?}"); + e + }) else { + return invalid_pointer_file(); + }; + if file_meta.len() > POINTER_FILE_LIMIT as u64 { + debug!("filesize: {}", file_meta.len()); + return invalid_pointer_file(); + } + let Ok(contents) = fs::read_to_string(path).map_err(|e| { + debug!("fs:read_to_string failed: {e:?}"); + e + }) else { + return invalid_pointer_file(); }; PointerFile::init_from_string(&contents, path) @@ -173,6 +196,9 @@ impl PointerFile { } } + pub fn path(&self) -> &str { + &self.path + } pub fn filesize(&self) -> u64 { self.filesize } @@ -214,6 +240,22 @@ impl std::fmt::Display for PointerFile { } } +// Check pointer file size limit at compile time. +// A valid pointer file looks like below +// +// # xet version 0 +// filesize = +// hash = '<64 digit long string>' +// +// +const_assert!( + POINTER_FILE_LIMIT + >= HEADER_PREFIX.len() + CURRENT_VERSION.len() // header ++ "filesize = ".len() + "9223372036854775807".len() // the largest i64 ++ "hash = ".len() + 64 + 2 // 2 is the single quotes size ++ 2 * 3 // 3 "\n" or "\r\n" on Windows +); + /// Tries to parse a pointer file from the reader, but if parsing fails, returns /// the data we have pulled out from the reader. (The AsyncIterator does not /// provide a "put-back" function and for simplicity it probably shouldn't @@ -259,6 +301,31 @@ pub async fn pointer_file_from_reader( } } +/// Smudge a pointer file and overwrite itself +pub async fn smudge_pointerfile_to_itself( + translator: &PointerFileTranslator, + path: &Path, +) -> anyhow::Result<()> { + let pointer_file = PointerFile::init_from_path(path); + + // not a pointer file, leave it as it is. + if !pointer_file.is_valid() { + return Ok(()); + } + + let file_hash = pointer_file.hash()?; + + // Create a temporary path for writing to, then move it over when done. + + let mut writer = Box::new(BufWriter::new(File::create(path)?)); + + translator + .smudge_file_from_hash(Some(path.to_owned()), &file_hash, &mut writer, None) + .await?; + + Ok(()) +} + #[cfg(test)] mod tests { const POINTER_FILE_VERSION: &str = "0"; diff --git a/rust/gitxetcore/src/data/remote_shard_interface.rs b/rust/gitxetcore/src/data/remote_shard_interface.rs index aceba9de..e4753feb 100644 --- a/rust/gitxetcore/src/data/remote_shard_interface.rs +++ b/rust/gitxetcore/src/data/remote_shard_interface.rs @@ -172,7 +172,9 @@ impl RemoteShardInterface { smudge_query_policy: config.smudge_query_policy, shard_manager, shard_client, - reconstruction_cache: Mutex::new(LruCache::new(FILE_RECONSTRUCTION_CACHE_SIZE)), + reconstruction_cache: Mutex::new(LruCache::new( + std::num::NonZero::new(FILE_RECONSTRUCTION_CACHE_SIZE).unwrap(), + )), cas, shard_downloads: Arc::new(singleflight::Group::new()), })) diff --git a/rust/gitxetcore/src/environment/mod.rs b/rust/gitxetcore/src/environment/mod.rs index cd05e86b..f574a518 100644 --- a/rust/gitxetcore/src/environment/mod.rs +++ b/rust/gitxetcore/src/environment/mod.rs @@ -1,4 +1,7 @@ pub mod axe; pub mod log; +pub mod process_utils; pub mod query_cache; pub mod upgrade_checks; + +pub use process_utils::CommandSanitized; diff --git a/rust/gitxetcore/src/environment/process_utils.rs b/rust/gitxetcore/src/environment/process_utils.rs new file mode 100644 index 00000000..b2b76c77 --- /dev/null +++ b/rust/gitxetcore/src/environment/process_utils.rs @@ -0,0 +1,24 @@ +use std; +use tokio; + +pub trait CommandSanitized { + fn new_sanitized(program: &str) -> Self; +} + +impl CommandSanitized for std::process::Command { + fn new_sanitized(program: &str) -> Self { + let mut command = std::process::Command::new(program); + command.env("LD_PRELOAD", ""); + command.env("DYLD_INSERT_LIBRARIES", ""); + command + } +} + +impl CommandSanitized for tokio::process::Command { + fn new_sanitized(program: &str) -> Self { + let mut command = tokio::process::Command::new(program); + command.env("LD_PRELOAD", ""); + command.env("DYLD_INSERT_LIBRARIES", ""); + command + } +} diff --git a/rust/gitxetcore/src/git_integration/git_process_wrapping.rs b/rust/gitxetcore/src/git_integration/git_process_wrapping.rs index 11ddf260..0a4d5512 100644 --- a/rust/gitxetcore/src/git_integration/git_process_wrapping.rs +++ b/rust/gitxetcore/src/git_integration/git_process_wrapping.rs @@ -1,3 +1,4 @@ +use crate::environment::CommandSanitized; use crate::errors::GitXetRepoError; use crate::errors::Result; use itertools::Itertools; @@ -36,7 +37,7 @@ fn spawn_git_command( ) -> Result { let git_executable = get_git_executable(); - let mut cmd = Command::new(git_executable); + let mut cmd = Command::new_sanitized(git_executable); cmd.arg(command).args(args); // Disable version check on recursive calls diff --git a/rust/gitxetcore/src/git_integration/git_repo_paths.rs b/rust/gitxetcore/src/git_integration/git_repo_paths.rs index 84c925de..94418b46 100644 --- a/rust/gitxetcore/src/git_integration/git_repo_paths.rs +++ b/rust/gitxetcore/src/git_integration/git_repo_paths.rs @@ -1,29 +1,9 @@ use crate::errors::Result; -use crate::git_integration::git_process_wrapping::run_git_captured; +use git2::Repository; +use path_absolutize::Absolutize; use std::path::PathBuf; -use std::str::FromStr; use tracing::info; -/// Returns true if the path is a bare repo -pub fn is_bare_repo(start_path: Option) -> Result { - let start_path = match start_path { - Some(p) => p, - None => std::env::current_dir()?, - }; - // check if this is a bare repo - let capture = run_git_captured( - Some(&start_path), - "rev-parse", - &["--is-bare-repository"], - false, - None, - ); - if let Ok((_, stdout, _)) = capture { - Ok(stdout == "true") - } else { - Ok(false) - } -} /// Returns the path of the repository we're operating in. /// /// If start_path is given, begin the search from there; otherwise start from the current working directory. @@ -32,48 +12,26 @@ pub fn is_bare_repo(start_path: Option) -> Result { /// /// If this is a bare repo, return_gitdir is irrelevant; both return_gitdir == true or false /// will return the same path. -pub fn resolve_repo_path( - start_path: Option, - return_gitdir: bool, -) -> Result> { +fn resolve_repo_path(start_path: Option, return_gitdir: bool) -> Result> { let start_path = match start_path { Some(p) => p, None => std::env::current_dir()?, }; - // --show-toplevel is fatal for a bare repo - let is_bare = is_bare_repo(Some(start_path.clone()))?; - - let (err_code, stdout, stderr) = run_git_captured( - Some(&start_path), - "rev-parse", - &[if return_gitdir || is_bare { - "--git-dir" - } else { - "--show-toplevel" - }], - false, - None, - )?; - - if let Some(0) = err_code { - let repo_path = PathBuf::from_str(stdout.trim()).unwrap(); - - let repo_path = if repo_path.is_absolute() { - repo_path - } else { - start_path.join(repo_path) - }; + let Ok(repo) = Repository::discover(&start_path).map_err(|e| { + info!("Error discovering repo from {start_path:?} : {e:?}"); + e + }) else { + return Ok(None); + }; - info!("Resolved git repo directory to {:?}.", &repo_path); - debug_assert!(repo_path.exists()); - Ok(Some(repo_path)) + if return_gitdir || repo.is_bare() { + Ok(repo.path().absolutize().map(|p| Some(p.to_path_buf()))?) } else { - info!( - "Resolving git repo failed with error code {:?}, stdout = {:?}, stderr = {:?}.", - &err_code, &stdout, &stderr - ); - Ok(None) + Ok(repo + .workdir() + .and_then(|p| p.absolutize().ok()) + .map(|p| p.to_path_buf())) } } @@ -90,17 +48,3 @@ pub fn get_repo_path(start_path: Option) -> Result> { pub fn get_git_path(start_path: Option) -> Result> { resolve_repo_path(start_path, true) } - -/// Given a repo directory, determine the git path reliably. -pub fn get_git_dir_from_repo_path(repo_path: &PathBuf) -> Result { - let git_path = PathBuf::from_str( - &run_git_captured(Some(repo_path), "rev-parse", &["--git-dir"], true, None)?.1, - ) - .unwrap(); - - Ok(if git_path.is_absolute() { - git_path - } else { - repo_path.join(git_path) - }) -} diff --git a/rust/gitxetcore/src/xetmnt/watch/metadata/cache.rs b/rust/gitxetcore/src/xetmnt/watch/metadata/cache.rs index 7d840758..15a00070 100644 --- a/rust/gitxetcore/src/xetmnt/watch/metadata/cache.rs +++ b/rust/gitxetcore/src/xetmnt/watch/metadata/cache.rs @@ -16,7 +16,9 @@ impl StatCache { /// Creates a new StatCache with the given capacity. pub fn new(capacity: usize) -> Self { Self { - cache: Mutex::new(LruCache::new(capacity)), + cache: Mutex::new(LruCache::new( + std::num::NonZero::new(capacity.max(1)).unwrap(), + )), } } diff --git a/rust/gitxetcore/src/xetmnt/xetfs_bare.rs b/rust/gitxetcore/src/xetmnt/xetfs_bare.rs index 8e047518..250e6492 100644 --- a/rust/gitxetcore/src/xetmnt/xetfs_bare.rs +++ b/rust/gitxetcore/src/xetmnt/xetfs_bare.rs @@ -395,7 +395,9 @@ impl XetFSBare { rootdir: 0, srcpath: srcpath.to_path_buf(), pfilereader: pfile, - statcache: RwLock::new(LruCache::new(STAT_CACHE_SIZE)), + statcache: RwLock::new(LruCache::new( + std::num::NonZero::new(STAT_CACHE_SIZE).unwrap(), + )), repo: tokio::sync::Mutex::new(repo), gitref: reference.into(), metadata, diff --git a/xetldfs/Cargo.lock b/xetldfs/Cargo.lock new file mode 100644 index 00000000..488c6904 --- /dev/null +++ b/xetldfs/Cargo.lock @@ -0,0 +1,5043 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if 1.0.0", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "async-scoped" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7a6a57c8aeb40da1ec037f5d455836852f7a57e69e1b1ad3d8f38ac1d6cadf" +dependencies = [ + "futures", + "pin-project", + "slab", + "tokio", +] + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "async-trait" +version = "0.1.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "atoi" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.29", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "binary-heap-plus" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4551d8382e911ecc0d0f0ffb602777988669be09447d536ff4388d1def11296" +dependencies = [ + "compare", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + +[[package]] +name = "blake3" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if 1.0.0", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "bytecount" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" + +[[package]] +name = "bytemuck" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + +[[package]] +name = "bytestream" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04f720842a717d6afaf69fee2dc69b771edc165f12cc3eb1b0e8eeef53a86454" +dependencies = [ + "byteorder", +] + +[[package]] +name = "cache" +version = "0.14.5" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.13.1", + "byteorder", + "chrono", + "lazy_static", + "lru", + "mockall", + "prometheus", + "tempfile", + "tokio", + "tracing", + "tracing-attributes", + "tracing-futures", + "tracing-subscriber", + "tracing-test", + "utils", + "xet_error", +] + +[[package]] +name = "cas_client" +version = "0.14.5" +dependencies = [ + "anyhow", + "async-trait", + "bincode", + "bytes", + "cache", + "clap 2.34.0", + "common_constants", + "deadpool", + "error_printer", + "futures", + "http 0.2.12", + "http-body-util", + "hyper 1.3.1", + "hyper-rustls", + "hyper-util", + "itertools 0.10.5", + "lazy_static", + "lz4", + "merkledb", + "merklehash", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-jaeger", + "parutils", + "progress_reporting", + "prost", + "retry_strategy", + "rustls-pemfile 2.1.2", + "serde_json", + "tempfile", + "tokio", + "tokio-native-tls", + "tokio-retry", + "tokio-rustls 0.25.0", + "tonic", + "tower", + "tracing", + "tracing-opentelemetry", + "utils", + "uuid", + "xet_error", +] + +[[package]] +name = "cc" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.52.5", +] + +[[package]] +name = "chunkpipe" +version = "0.14.5" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim 0.8.0", + "textwrap 0.11.0", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_derive", + "clap_lex", + "indexmap 1.9.3", + "once_cell", + "strsim 0.10.0", + "termcolor", + "textwrap 0.16.1", +] + +[[package]] +name = "clap_derive" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "color-eyre" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] + +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + +[[package]] +name = "common_constants" +version = "0.14.5" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "compare" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120133d4db2ec47efe2e26502ee984747630c67f51974fca0b6c1340cf2368d3" + +[[package]] +name = "config" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23738e11972c7643e4ec947840fc463b6a571afcd3e735bdfce7d03c7a784aca" +dependencies = [ + "async-trait", + "json5", + "lazy_static", + "nom", + "pathdiff", + "ron 0.7.1", + "rust-ini", + "serde", + "serde_json", + "toml", + "yaml-rust", +] + +[[package]] +name = "const_format" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a214c7af3d04997541b18d432afaff4c455e79e2029079647e72fc2bd27673" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f6ff08fd20f4f299298a28e2dfa8a8ba1036e6cd2460ac1de7b425d76f2500" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crossterm" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" +dependencies = [ + "bitflags 1.3.2", + "crossterm_winapi", + "libc", + "mio", + "parking_lot 0.12.3", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ctrlc" +version = "3.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" +dependencies = [ + "nix", + "windows-sys 0.52.0", +] + +[[package]] +name = "cxx" +version = "1.0.124" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273dcfd3acd4e1e276af13ed2a43eea7001318823e7a726a6b3ed39b4acc0b82" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.124" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b2766fbd92be34e9ed143898fce6c572dc009de39506ed6903e5a05b68914e" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.67", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.124" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839fcd5e43464614ffaa989eaf1c139ef1f0c51672a1ed08023307fa1b909ccd" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.124" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2c1c1776b986979be68bb2285da855f8d8a35851a769fca8740df7c3d07877" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "data_analysis" +version = "0.14.5" +dependencies = [ + "approx", + "cxx", + "cxx-build", + "float-cmp", + "serde", + "truncrate", +] + +[[package]] +name = "deadpool" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e" +dependencies = [ + "async-trait", + "deadpool-runtime", + "num_cpus", + "retain_mut", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +dependencies = [ + "tokio", +] + +[[package]] +name = "deadqueue" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a2561fd313df162315935989dceb8c99db4ee1933358270a57a3cfb8c957f3" +dependencies = [ + "crossbeam-queue", + "tokio", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common", +] + +[[package]] +name = "dirs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" +dependencies = [ + "cfg-if 0.1.10", + "dirs-sys", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "either" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" + +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "error_printer" +version = "0.14.5" +dependencies = [ + "tracing", +] + +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + +[[package]] +name = "file_utils" +version = "0.14.2" +dependencies = [ + "anyhow", + "colored", + "lazy_static", + "libc", + "rand 0.8.5", + "tempfile", + "tracing", + "whoami", + "winapi", +] + +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "gearhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8cf82cf76cd16485e56295a1377c775ce708c9f1a0be6b029076d60a245d213" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "git-url-parse" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b037f7449dd4a8b711e660301ff1ff28aa00eea09698421fb2d78db51a7b7a72" +dependencies = [ + "color-eyre", + "regex", + "strum", + "strum_macros", + "tracing", + "url", +] + +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "git2" +version = "0.18.2" +source = "git+https://github.com/xetdata/git2-rs#dfffd3d1eb9e87a9e7e98b24d0a0f54db664cbcc" +dependencies = [ + "bitflags 2.5.0", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + +[[package]] +name = "gitxetcore" +version = "0.14.5" +dependencies = [ + "anyhow", + "async-trait", + "atoi", + "atty", + "base64 0.13.1", + "bincode", + "blake3", + "cas_client", + "chrono", + "chunkpipe", + "clap 3.2.25", + "colored", + "common_constants", + "const_format", + "csv-core", + "ctrlc", + "data_analysis", + "dirs 4.0.0", + "enum_dispatch", + "error_printer", + "fallible-iterator", + "file_utils", + "filetime", + "futures", + "futures-core", + "git-url-parse", + "git-version", + "git2", + "glob", + "hashers", + "hex", + "http 0.2.12", + "humantime", + "intaglio", + "is_executable", + "itertools 0.10.5", + "lazy", + "lazy_static", + "libc", + "libmagic", + "lru", + "lz4", + "mdb_shard", + "merkledb", + "merklehash", + "mockall", + "mockall_double", + "more-asserts", + "nfsserve", + "normalize-path", + "openssl", + "openssl-probe", + "opentelemetry", + "opentelemetry-jaeger", + "parutils", + "path-absolutize", + "pathdiff", + "pbr", + "progress_reporting", + "prometheus", + "prometheus_dict_encoder", + "rand 0.8.5", + "regex", + "reqwest", + "retry_strategy", + "ring 0.16.20", + "same-file", + "serde", + "serde_json", + "serde_with", + "shard_client", + "shellexpand", + "slog", + "slog-async", + "slog-json", + "snailquote", + "sorted-vec", + "static_assertions", + "sysinfo", + "tableau_summary", + "tabled", + "tempdir", + "tempfile", + "tokio", + "toml", + "tracing", + "tracing-attributes", + "tracing-futures", + "tracing-opentelemetry", + "tracing-subscriber", + "tracing-test", + "url", + "utils", + "utime", + "uuid", + "version-compare", + "walkdir", + "whoami", + "winapi", + "xet_config", + "xet_error", +] + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash 0.8.11", + "allocator-api2", +] + +[[package]] +name = "hashers" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2bca93b15ea5a746f220e56587f71e73c6165eab783df9e26590069953e3c30" +dependencies = [ + "fxhash", +] + +[[package]] +name = "hashring" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2e670d8fa425ec0d91dae7d6ab4a32721e775060a5d2d7cd572a9f0736dfddc" +dependencies = [ + "siphasher", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "heed" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269c7486ed6def5d7b59a427cec3e87b4d4dd4381d01e21c8c9f2d3985688392" +dependencies = [ + "bytemuck", + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-rkv-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", +] + +[[package]] +name = "heed-traits" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a53a94e5b2fd60417e83ffdfe136c39afacff0d4ac1d8d01cd66928ac610e1a2" + +[[package]] +name = "heed-types" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a6cf0a6952fcedc992602d5cddd1e3fff091fbe87d38636e3ec23a31f32acbd" +dependencies = [ + "bincode", + "bytemuck", + "byteorder", + "heed-traits", + "serde", + "serde_json", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.3.1", + "hyper-util", + "log", + "rustls 0.22.4", + "rustls-native-certs 0.7.0", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.25.0", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.29", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.29", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "hyper-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "imara-diff" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e98c1d0ad70fc91b8b9654b1f33db55e59579d3b3de2bffdced0fdb810570cb8" +dependencies = [ + "ahash 0.8.11", + "hashbrown 0.12.3", +] + +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "intaglio" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b4f756a47a2dac507018af2d4e47988e93829f34a665da3655b23cc1d21ee47" + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "is_executable" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa9acdc6d67b75e626ad644734e8bc6df893d9cd2a834129065d3dd6158ea9c8" +dependencies = [ + "winapi", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "lazy" +version = "0.14.5" +dependencies = [ + "lazy_static", + "tokio", + "tracing", + "xet_error", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "len-trait" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723558ab8acaa07cb831b424cd164b587ddc1648b34748a30953c404e9a4a65b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "libgit2-sys" +version = "0.16.2+1.7.2" +source = "git+https://github.com/xetdata/git2-rs#dfffd3d1eb9e87a9e7e98b24d0a0f54db664cbcc" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libmagic" +version = "0.14.5" +dependencies = [ + "anyhow", + "phf", + "serde", + "serde_json", + "tracing", + "tracing-attributes", + "tracing-subscriber", +] + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.5.0", + "libc", +] + +[[package]] +name = "libxet" +version = "0.14.5" +dependencies = [ + "error_printer", + "gitxetcore", + "merkledb", + "progress_reporting", +] + +[[package]] +name = "libz-sys" +version = "1.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "link-cplusplus" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" +dependencies = [ + "cc", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "lmdb-rkv-sys" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "lru" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "lz4" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6eab492fe7f8651add23237ea56dbf11b3c4ff762ab83d40a47f11433421f91" +dependencies = [ + "libc", + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9764018d143cc854c9f17f0b907de70f14393b1f502da6375dce70f00514eb3" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "mdb_shard" +version = "0.14.5" +dependencies = [ + "anyhow", + "async-scoped", + "async-trait", + "binary-heap-plus", + "clap 3.2.25", + "lazy_static", + "merkledb", + "merklehash", + "more-asserts", + "rand 0.8.5", + "regex", + "serde", + "tempdir", + "tempfile", + "tokio", + "tracing", + "uuid", + "xet_error", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "merkledb" +version = "0.14.5" +dependencies = [ + "async-trait", + "bincode", + "bitflags 1.3.2", + "blake3", + "clap 3.2.25", + "futures", + "gearhash", + "itertools 0.10.5", + "lazy_static", + "merklehash", + "parutils", + "rand 0.8.5", + "rand_chacha", + "rand_core 0.6.4", + "rayon", + "ron 0.6.6", + "rustc-hash", + "serde", + "structopt", + "tempfile", + "tokio", + "tracing", + "walkdir", + "xet_error", +] + +[[package]] +name = "merklehash" +version = "0.14.5" +dependencies = [ + "blake3", + "generic-array", + "heed", + "rand 0.8.5", + "rand_chacha", + "rand_core 0.6.4", + "safe-transmute", + "serde", + "sha3", + "structopt", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "mockall" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +dependencies = [ + "cfg-if 1.0.0", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "mockall_double" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1ca96e5ac35256ae3e13536edd39b172b88f41615e1d7b653c8ad24524113e8" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "more-asserts" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" + +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" + +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nfsserve" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d73615e054238e6bf5e554407b5b23e82fc63616db459057c51b794799eda6fb" +dependencies = [ + "anyhow", + "async-trait", + "byteorder", + "bytestream", + "filetime", + "futures", + "num-derive", + "num-traits", + "smallvec", + "tokio", + "tracing", + "tracing-attributes", +] + +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.5.0", + "cfg-if 1.0.0", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "normalize-path" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c11b7fa387f3c9874e60670ac6d009cefc5ffa8c23437137a9998c0a154e77" + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-src" +version = "300.3.1+3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7259953d42a81bf137fbbd73bd30a8e1914d6dce43c2b90ed575783a22608b91" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "opentelemetry" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "js-sys", + "lazy_static", + "percent-encoding", + "pin-project", + "rand 0.8.5", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "opentelemetry-http" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449048140ee61e28f57abe6e9975eedc1f3a29855c7407bd6c12b18578863379" +dependencies = [ + "async-trait", + "bytes", + "http 0.2.12", + "opentelemetry", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c0b12cd9e3f9b35b52f6e0dac66866c519b26f424f4bbf96e3fe8bfbdc5229" +dependencies = [ + "async-trait", + "lazy_static", + "opentelemetry", + "opentelemetry-semantic-conventions", + "thiserror", + "thrift", + "tokio", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985cc35d832d412224b2cffe2f9194b1b89b6aa5d0bef76d080dce09d90e62bd" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "ordered-float" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list", + "hashbrown 0.12.3", +] + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + +[[package]] +name = "page_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "papergrid" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae7891b22598926e4398790c8fe6447930c72a67d36d983a49d6ce682ce83290" +dependencies = [ + "bytecount", + "fnv", + "unicode-width", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.10", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.5.2", + "smallvec", + "windows-targets 0.52.5", +] + +[[package]] +name = "parutils" +version = "0.14.5" +dependencies = [ + "anyhow", + "async-scoped", + "async-trait", + "deadqueue", + "futures", + "len-trait", + "more-asserts", + "tokio", + "tracing", +] + +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + +[[package]] +name = "pbr" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5827dfa0d69b6c92493d6c38e633bbaa5937c153d0d7c28bf12313f8c6d514" +dependencies = [ + "crossbeam-channel", + "libc", + "winapi", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pest" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "pest_meta" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.2.6", +] + +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "predicates" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" +dependencies = [ + "difflib", + "float-cmp", + "itertools 0.10.5", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +dependencies = [ + "proc-macro2", + "syn 2.0.67", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "progress_reporting" +version = "0.14.5" +dependencies = [ + "atty", + "crossterm", + "more-asserts", + "tokio", + "tracing", + "utils", +] + +[[package]] +name = "prometheus" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "memchr", + "parking_lot 0.12.3", + "protobuf", + "thiserror", +] + +[[package]] +name = "prometheus_dict_encoder" +version = "0.14.5" +dependencies = [ + "memchr", + "prometheus", + "tracing", +] + +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +dependencies = [ + "bytes", + "heck 0.5.0", + "itertools 0.12.1", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.67", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redhook" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6e109b8469dbbe6d7cbe18e5ebd65d4a134e2f4b91304ad9213984cad1d70a6" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +dependencies = [ + "bitflags 2.5.0", +] + +[[package]] +name = "redox_users" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.29", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "retain_mut" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" + +[[package]] +name = "retry_strategy" +version = "0.14.5" +dependencies = [ + "tokio-retry", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted 0.7.1", + "web-sys", + "winapi", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if 1.0.0", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + +[[package]] +name = "ron" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86018df177b1beef6c7c8ef949969c4f7cb9a9344181b92486b23c79995bdaa4" +dependencies = [ + "base64 0.13.1", + "bitflags 1.3.2", + "serde", +] + +[[package]] +name = "ron" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" +dependencies = [ + "base64 0.13.1", + "bitflags 1.3.2", + "serde", +] + +[[package]] +name = "roxmltree" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f" + +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if 1.0.0", + "ordered-multimap", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring 0.17.8", + "rustls-webpki 0.101.7", + "sct", +] + +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring 0.17.8", + "rustls-pki-types", + "rustls-webpki 0.102.4", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-native-certs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.1.2", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + +[[package]] +name = "rustls-webpki" +version = "0.102.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +dependencies = [ + "ring 0.17.8", + "rustls-pki-types", + "untrusted 0.9.0", +] + +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "safe-transmute" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scratch" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + +[[package]] +name = "security-framework" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +dependencies = [ + "bitflags 2.5.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "serde_json" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +dependencies = [ + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.7", +] + +[[package]] +name = "sha3" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "keccak", + "opaque-debug", +] + +[[package]] +name = "shard_client" +version = "0.14.5" +dependencies = [ + "anyhow", + "async-trait", + "bincode", + "bytes", + "cas_client", + "clap 2.34.0", + "heed", + "http 0.2.12", + "hyper 0.14.29", + "itertools 0.10.5", + "lazy_static", + "mdb_shard", + "merkledb", + "merklehash", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-jaeger", + "progress_reporting", + "prost", + "retry_strategy", + "serde_json", + "tempfile", + "tokio", + "tokio-retry", + "tonic", + "tower", + "tracing", + "tracing-opentelemetry", + "utils", + "uuid", + "xet_error", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shellexpand" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c7e79eddc7b411f9beeaaf2d421de7e7cb3b1ab9eaf1b79704c0e4130cba6b5" +dependencies = [ + "dirs 2.0.2", +] + +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-async" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" +dependencies = [ + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", +] + +[[package]] +name = "slog-json" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" +dependencies = [ + "serde", + "serde_json", + "slog", + "time", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "snailquote" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec62a949bda7f15800481a711909f946e1204f2460f89210eaf7f57730f88f86" +dependencies = [ + "thiserror", + "unicode_categories", +] + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "sorted-vec" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6734caf0b6f51addd5eeacca12fb39b2c6c14e8d4f3ac42f3a78955c0467458" + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap 2.34.0", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck 0.3.3", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "subtle" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0208408ba0c3df17ed26eb06992cb1a1268d41b2c0e12e65203fbe3972cee5" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "synchronoise" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" +dependencies = [ + "crossbeam-queue", +] + +[[package]] +name = "sysinfo" +version = "0.26.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c18a6156d1f27a9592ee18c1a846ca8dd5c258b7179fc193ae87c74ebb666f5" +dependencies = [ + "cfg-if 1.0.0", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tableau_summary" +version = "0.14.5" +dependencies = [ + "anyhow", + "error_printer", + "imara-diff", + "itertools 0.12.1", + "once_cell", + "regex", + "roxmltree", + "serde", + "serde_json", + "tracing", + "url", +] + +[[package]] +name = "tabled" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce69a5028cd9576063ec1f48edb2c75339fd835e6094ef3e05b3a079bf594a6" +dependencies = [ + "papergrid", + "tabled_derive", + "unicode-width", +] + +[[package]] +name = "tabled_derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99f688a08b54f4f02f0a3c382aefdb7884d3d69609f785bd253dc033243e3fe4" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if 1.0.0", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + +[[package]] +name = "thiserror" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "thrift" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b82ca8f46f95b3ce96081fe3dd89160fdea970c254bb72925255d1b62aae692e" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float", + "threadpool", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot 0.12.3", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "tonic" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.7", + "bytes", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.29", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "rustls-pemfile 1.0.4", + "tokio", + "tokio-rustls 0.24.1", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbbe89715c1dbbb790059e2565353978564924ee85017b5fff365c872ff6721f" +dependencies = [ + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log 0.1.4", + "tracing-subscriber", +] + +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log 0.2.0", + "tracing-serde", +] + +[[package]] +name = "tracing-test" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "557b891436fe0d5e0e363427fc7f217abf9ccd510d5136549847bdcbcd011d68" +dependencies = [ + "tracing-core", + "tracing-subscriber", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" +dependencies = [ + "quote", + "syn 2.0.67", +] + +[[package]] +name = "truncrate" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73eead03a57feec88e556e6be8c3f6c9917688a15f9d410aedee1b6cb276445f" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode-width" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utils" +version = "0.14.5" +dependencies = [ + "anyhow", + "chrono", + "futures", + "hashbrown 0.12.3", + "hashring", + "lazy_static", + "merklehash", + "parking_lot 0.11.2", + "pin-project", + "prost", + "prost-types", + "regex", + "serde", + "tempfile", + "tokio", + "tonic", + "tonic-build", + "tracing", + "xet_error", +] + +[[package]] +name = "utime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91baa0c65eabd12fcbdac8cc35ff16159cab95cae96d0222d6d0271db6193cef" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "uuid" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +dependencies = [ + "atomic", + "getrandom", + "rand 0.8.5", +] + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.67", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall 0.4.1", + "wasite", + "web-sys", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if 1.0.0", + "windows-sys 0.48.0", +] + +[[package]] +name = "xet-error-impl" +version = "1.0.50" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "xet_config" +version = "0.14.5" +dependencies = [ + "config", + "dirs 4.0.0", + "serde", + "toml", + "tracing", + "xet_error", +] + +[[package]] +name = "xet_error" +version = "0.14.5" +dependencies = [ + "lazy_static", + "xet-error-impl", +] + +[[package]] +name = "xetldfs" +version = "0.14.2" +dependencies = [ + "anyhow", + "clap 3.2.25", + "ctor", + "errno", + "file_utils", + "lazy_static", + "libc", + "libxet", + "openssl-probe", + "redhook", + "regex", + "tempdir", + "tempfile", + "tokio", + "tracing", +] + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "zerocopy" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/xetldfs/Cargo.toml b/xetldfs/Cargo.toml new file mode 100644 index 00000000..977ea160 --- /dev/null +++ b/xetldfs/Cargo.toml @@ -0,0 +1,52 @@ +[package] +name = "xetldfs" +version = "0.14.2" +edition = "2021" + +[[bin]] +name = "x" +path = "src/bin/xcmd.rs" + +[[bin]] +name = "git-xet" +path = "../gitxet/src/bin/gitxet.rs" + +[lib] +name = "xetldfs" +crate-type = ["dylib"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +lazy_static = "*" +libc = "0.2.155" +libxet = { path = "../libxet" } +file_utils = { path = "../rust/file_utils" } + +anyhow = "1" +tokio = { version = "1", features = ["full"] } +errno = "0.3.9" +ctor = "0.1" +redhook = "*" +openssl-probe = "0.1.5" +clap = { version = "3.1.6", features = ["derive"] } + +[dev-dependencies] +anyhow = "1" +tempdir = "0.3" +tempfile = "3" +regex = "1.5.6" +tracing = "0.1.*" + +[profile.release] +opt-level = 3 +lto = true +debug = 1 + +[profile.opt-test] +inherits = "dev" +opt-level = 1 +debug = 1 + +[features] +openssl_vendored = ["libxet/openssl_vendored"] + diff --git a/xetldfs/src/bin/xcmd.rs b/xetldfs/src/bin/xcmd.rs new file mode 100644 index 00000000..7a87b96f --- /dev/null +++ b/xetldfs/src/bin/xcmd.rs @@ -0,0 +1,320 @@ +use clap::{Args, Parser, Subcommand}; +use libc::*; +use std::fs::File; +use std::io::{self, Read, Seek, Write}; +use std::path::{Path, PathBuf}; + +#[derive(Parser)] +struct XCommand { + #[clap(subcommand)] + command: Command, +} + +impl XCommand { + fn run(&self) -> anyhow::Result<()> { + self.command.run() + } +} + +#[derive(Subcommand)] +enum Command { + /// Equivalent to "cat". + Cat(MultiplePathArg), + + /// Write a str from stdin to one or more files, replacing the contents if the file exists. + Write(MultiplePathArg), + + /// Write a str from stdin to a file, appending it to the end if the file exists. + Append(PathArg), + + /// Write a str from stdin to a file at a specific location, rewrite the contents + /// at that location if the file exists. If the specified location is + /// beyond the original file size, the gap between the previous file end + /// and this location is untouched, the behavior of reading from this gap + /// is platform dependent. + Writeat(WriteatArgs), + + /// Get file size by calling "stat". + Stat(PathArg), + + /// Get file size by calling "fstat". + Fstat(PathArg), + + /// Equivalent to "cat" but uses mmap. + CatMmap(MultiplePathArg), + + /// Same as Write, but uses mmap + WriteMmap(MultiplePathArg), + + /// Equivalent to Writeat but uses mmap + WriteatMmap(WriteatArgs), + + /// Equivalent to Append but uses mmap + AppendMmap(PathArg), + + /// Testing things + Verify, +} + +#[derive(Args)] +struct PathArg { + file: PathBuf, +} + +#[derive(Args)] +struct MultiplePathArg { + files: Vec, +} + +#[derive(Args)] +struct WriteatArgs { + pos: u64, + file: PathBuf, +} + +impl Command { + pub fn run(&self) -> anyhow::Result<()> { + match self { + Command::Cat(args) => cat(&args.files), + Command::Write(args) => write(&args.files), + Command::Append(args) => append(&args.file), + Command::Writeat(args) => writeat(args), + Command::Stat(args) => stat(&args.file), + Command::Fstat(args) => fstat(&args.file), + Command::CatMmap(args) => cat_mmap(&args.files), + Command::WriteMmap(args) => write_mmap(&args.files), + Command::WriteatMmap(args) => writeat_mmap(args), + Command::AppendMmap(args) => append_mmap(&args.file), + Command::Verify => { + println!("VERIFICATION"); + Ok(()) + } + } + } +} + +fn read_from_stdin() -> io::Result> { + let mut buffer = Vec::new(); + let mut stdin = io::stdin(); + stdin.read_to_end(&mut buffer)?; + Ok(buffer) +} + +fn cat(files: &Vec) -> anyhow::Result<()> { + for path in files { + let contents = std::fs::read(path)?; + let mut stdout = io::stdout(); + stdout.write_all(&contents)?; + } + + Ok(()) +} + +fn write(files: &Vec) -> anyhow::Result<()> { + let data = read_from_stdin()?; + + for path in files { + std::fs::write(path, &data)?; + } + + Ok(()) +} + +fn append(path: impl AsRef) -> anyhow::Result<()> { + let data = read_from_stdin()?; + + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + + file.write_all(&data)?; + + Ok(()) +} + +fn writeat(args: &WriteatArgs) -> anyhow::Result<()> { + let data = read_from_stdin()?; + + let mut file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(&args.file)?; + + file.seek(io::SeekFrom::Start(args.pos))?; + file.write_all(&data)?; + + Ok(()) +} + +fn stat(path: impl AsRef) -> anyhow::Result<()> { + let meta = std::fs::metadata(path)?; + + println!("{}", meta.len()); + + Ok(()) +} + +fn fstat(path: impl AsRef) -> anyhow::Result<()> { + let file = std::fs::OpenOptions::new().read(true).open(path)?; + let meta = file.metadata()?; + + println!("{}", meta.len()); + + Ok(()) +} + +use libc::{c_void, mmap, munmap, size_t, MAP_PRIVATE, PROT_READ}; + +use std::os::unix::io::AsRawFd; + +fn read_file_with_mmap(file_path: impl AsRef) -> std::io::Result> { + // Open the file + let file = File::open(file_path)?; + let fd = file.as_raw_fd(); + + // Get the file length + let metadata = file.metadata()?; + let file_length = metadata.len() as size_t; + + // Memory map the file + let mmap_addr = unsafe { + mmap( + std::ptr::null_mut(), + file_length, + PROT_READ, + MAP_PRIVATE, + fd, + 0, + ) as *mut u8 + }; + + if mmap_addr == libc::MAP_FAILED as *mut u8 { + return Err(io::Error::last_os_error()); + } + + // Read the file content from the mapped memory + let mut buffer = Vec::with_capacity(file_length); + unsafe { + buffer.extend_from_slice(std::slice::from_raw_parts(mmap_addr, file_length)); + } + + // Unmap the memory + unsafe { + munmap(mmap_addr as *mut c_void, file_length); + } + + Ok(buffer) +} + +fn cat_mmap(files: &Vec) -> anyhow::Result<()> { + let mut out_data = Vec::>::new(); + + for path in files { + let data = read_file_with_mmap(path)?; + io::stdout().write_all(&data)?; + out_data.push(data); + } + + for (path, data) in files.iter().zip(out_data.iter()) { + eprintln!("{path:?}: {:?}", std::str::from_utf8(data).unwrap()); + } + + Ok(()) +} + +enum WriteMode { + WriteAtEnd, + WriteFromPos(usize), + + OverwriteFile, +} + +fn write_to_file_with_mmap( + file_path: impl AsRef, + write_at: WriteMode, + data: &[u8], +) -> std::io::Result<()> { + // Open the file + let file = File::options().read(true).write(true).open(file_path)?; + let fd = file.as_raw_fd(); + + // Get the file length + let metadata = file.metadata()?; + let mut file_length = metadata.len() as size_t; + + let (offset, new_file_length) = match write_at { + WriteMode::WriteAtEnd => (file_length, file_length + data.len()), + WriteMode::WriteFromPos(p) => (p, file_length.max(p + data.len())), + WriteMode::OverwriteFile => (0, data.len()), + }; + + if new_file_length != file_length { + file.set_len(new_file_length as u64)?; + file_length = new_file_length; + } + + // Memory map the file + let mmap_addr = unsafe { + mmap( + std::ptr::null_mut(), + file_length, + PROT_READ | PROT_WRITE, + MAP_SHARED, + fd, + 0, + ) as *mut u8 + }; + + if mmap_addr == libc::MAP_FAILED as *mut u8 { + return Err(io::Error::last_os_error()); + } + + // Write the data to the mapped memory + unsafe { + let mmap_slice = std::slice::from_raw_parts_mut(mmap_addr, file_length); + mmap_slice[offset..offset + data.len()].copy_from_slice(data); + } + + // Unmap the memory + unsafe { + munmap(mmap_addr as *mut c_void, file_length); + } + + Ok(()) +} + +fn write_mmap(files: &Vec) -> anyhow::Result<()> { + let data = read_from_stdin()?; + + for path in files { + write_to_file_with_mmap(path, WriteMode::OverwriteFile, &data)?; + std::fs::write(path, &data)?; + } + + Ok(()) +} + +fn writeat_mmap(args: &WriteatArgs) -> anyhow::Result<()> { + let data = read_from_stdin()?; + write_to_file_with_mmap( + &args.file, + WriteMode::WriteFromPos(args.pos as usize), + &data, + )?; + + Ok(()) +} + +fn append_mmap(path: impl AsRef) -> anyhow::Result<()> { + let data = read_from_stdin()?; + write_to_file_with_mmap(path, WriteMode::WriteAtEnd, &data)?; + + Ok(()) +} + +fn main() -> anyhow::Result<()> { + let cli = XCommand::parse(); + cli.run() +} diff --git a/xetldfs/src/errors.rs b/xetldfs/src/errors.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/xetldfs/src/errors.rs @@ -0,0 +1 @@ + diff --git a/xetldfs/src/lib.rs b/xetldfs/src/lib.rs new file mode 100644 index 00000000..e5505479 --- /dev/null +++ b/xetldfs/src/lib.rs @@ -0,0 +1,641 @@ +use libc::*; +use std::ffi::CStr; +use std::ptr::null_mut; +#[macro_use] +extern crate redhook; +extern crate libc; + +mod path_utils; +mod runtime; +mod utils; +mod xet_interface; +mod xet_rfile; + +use crate::path_utils::{is_regular_fd, is_regular_file, resolve_path_from_fd}; +use crate::runtime::{ + activate_fd_runtime, interposing_disabled, process_in_interposable_state, + with_interposing_disabled, +}; +use crate::utils::*; +use crate::xet_interface::{ + file_needs_materialization, materialize_file_under_fd_if_needed, materialize_rw_file_if_needed, +}; +use crate::xet_rfile::{ + close_fd_if_registered, maybe_fd_read_managed, register_interposed_read_fd, + set_fd_read_interpose, +}; + +#[allow(unused)] +use crate::utils::C_EMPTY_STR; + +#[ctor::ctor] +fn print_open() { + eprintln!("XetLDFS interposing library loaded."); +} + +pub const ENABLE_CALL_TRACING: bool = false; +pub const ENABLE_CALL_TRACING_FULL: bool = false; + +macro_rules! ld_func_trace { + ($func_name:expr, $($var:ident),*) => { + if ENABLE_CALL_TRACING_FULL { + if crate::runtime::raw_runtime_activated() { + eprint!("XetLDFS[{}] {}: ", unsafe {libc::getpid() }, $func_name); + $( + eprint!("{}={:?} ", stringify!($var), $var); + )* + eprintln!(); + } + } + }; +} + +#[macro_export] +macro_rules! ld_trace { + ($($arg:tt)*) => { + if ENABLE_CALL_TRACING { + if $crate::runtime::raw_runtime_activated() { + let text = format!($($arg)*); + eprintln!("XetLDFS[{}]: {text}", unsafe {libc::getpid() }); + } + } + }; +} + +#[macro_export] +macro_rules! ld_warn { + ($($arg:tt)*) => { + if $crate::runtime::runtime_activated() { + let text = format!($($arg)*); + eprintln!("XetLDFS WARNING: {text}"); + } + }; +} + +// 0666, copied from sys/stat.h +const DEFFILEMODE: mode_t = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; + +#[inline] +unsafe fn fopen_impl( + pathname: &str, + mode: *const c_char, + callback: impl Fn() -> *mut libc::FILE, +) -> *mut libc::FILE { + ld_func_trace!("fopen_impl", pathname, mode); + + // Convert fopen mode to OpenOptions + let mode_str = CStr::from_ptr(mode).to_str().unwrap(); + let Some(open_flags) = open_flags_from_mode_string(mode_str) else { + ld_warn!("Bad open flags? {mode_str}"); + return callback(); + }; + + if file_needs_materialization(open_flags) { + ld_trace!("fopen_impl: Materializing path {pathname}."); + materialize_rw_file_if_needed(pathname); + // no need to interpose a non-pointer file + return callback(); + } + + // only interpose read + if open_flags & O_ACCMODE == O_RDONLY { + let ret = callback(); + if ret.is_null() { + ld_trace!("fopen_impl: callback returned null."); + return null_mut(); + } + + let fd = fileno(ret); + register_interposed_read_fd(pathname, fd); + ld_trace!("fopen_impl: Registered {pathname} as read interposed with fd = {fd}."); + ret + } else { + ld_trace!("fopen_impl: passing through."); + callback() + } +} + +hook! { + unsafe fn fopen(pathname: *const c_char, mode: *const c_char) -> *mut libc::FILE => my_fopen { + ld_func_trace!("fopen", pathname, mode); + + if interposing_disabled() { return real!(fopen)(pathname, mode); } + + let _ig = with_interposing_disabled(); + + if !is_regular_file(pathname) { + // We only interpose for regular files (not for socket, symlink, block dev, directory, character device (/dev/tty), fifo). + return real!(fopen)(pathname, mode); + } + + let path = unsafe { c_to_str(pathname) }; + fopen_impl(path, mode, || real!(fopen)(pathname, mode)) + } +} + +#[cfg(target_os = "linux")] +hook! { + unsafe fn fopen64(pathname: *const c_char, mode: *const c_char) -> *mut libc::FILE => my_fopen64 { + ld_func_trace!("fopen64", pathname, mode); + if interposing_disabled() { return real!(fopen64)(pathname, mode); } + + let _ig = with_interposing_disabled(); + + let path = unsafe { c_to_str(pathname) }; + fopen_impl(path, mode, || real!(fopen64)(pathname, mode)) + } +} + +hook! { + unsafe fn freopen(path: *const libc::c_char, mode: *const libc::c_char, stream: *mut libc::FILE) -> *mut libc::FILE => my_freopen { + ld_func_trace!("freopen", path, mode); + if interposing_disabled() { return real!(freopen)(path, mode, stream); } + + let _ig = with_interposing_disabled(); + + let fd = fileno(stream); + + // Close this if registered. + if close_fd_if_registered(fd) { + my_fopen(path, mode) + } else { + real!(freopen)(path, mode, stream) + } + } +} + +#[inline] +unsafe fn open_impl(pathname: &str, open_flags: c_int, callback: impl Fn() -> c_int) -> c_int { + if file_needs_materialization(open_flags) { + materialize_rw_file_if_needed(pathname); + // no need to interpose a non-pointer file + return callback(); + } + + // only interpose read + if open_flags & O_ACCMODE == O_RDONLY { + ld_trace!("file {pathname} is RDONLY"); + let fd = callback(); + ld_trace!("file {pathname} is RDONLY, opened as {fd}"); + if fd != -1 { + register_interposed_read_fd(pathname, fd); + } + fd + } else { + callback() + } +} + +// Hook for open +hook! { + unsafe fn open(pathname: *const c_char, flags: c_int, filemode: mode_t) -> c_int => my_open { + let path = c_to_str(pathname); + ld_func_trace!("open", path, flags, filemode); + activate_fd_runtime(); + + if interposing_disabled() { + return real!(open)(pathname, flags, filemode); + } + + let _ig = with_interposing_disabled(); + + if !is_regular_file(pathname) { + // We only interpose for regular files (not for socket, symlink, block dev, directory, character device (/dev/tty), fifo). + return real!(open)(pathname, flags, filemode); + } + + let path = unsafe { c_to_str(pathname) }; + open_impl(path ,flags, || real!(open)(pathname, flags, filemode)) + } +} + +#[inline] +unsafe fn real_open(pathname: *const c_char, flags: c_int, filemode: mode_t) -> c_int { + real!(open)(pathname, flags, filemode) +} + +#[cfg(target_os = "linux")] +hook! { + unsafe fn open64(pathname: *const c_char, flags: c_int, filemode: mode_t) -> c_int => my_open64 { + let path = c_to_str(pathname); + ld_func_trace!("open64", path, flags, filemode); + activate_fd_runtime(); + + if interposing_disabled() { + return real!(open64)(pathname, flags, filemode); + } + + let _ig = with_interposing_disabled(); + + // We only interpose for regular files (not for socket, symlink, block dev, directory, character device (/dev/tty), fifo). + if !is_regular_file(pathname) { + return real!(open64)(pathname, flags, filemode); + } + + let path = unsafe { c_to_str(pathname) }; + open_impl(path, flags, || real!(open64)(pathname, flags, filemode)) + } +} + +hook! { + unsafe fn openat(dirfd: libc::c_int, pathname: *const libc::c_char, flags: libc::c_int, filemode : mode_t) -> libc::c_int => my_openat { + let path = c_to_str(pathname); + ld_func_trace!("openat", dirfd, path, filemode); + activate_fd_runtime(); + + if !interposing_disabled() { + + let _ig = with_interposing_disabled(); + + if let Some(path) = resolve_path_from_fd(dirfd, pathname) { + + // We only interpose for regular files (not for socket, symlink, block dev, directory, character device (/dev/tty), fifo). + if is_regular_file(path.as_ptr()) { + ld_trace!("openat: Path {} is regular file, calling open_impl (dirfd={dirfd}, pathname={})", path.to_str().unwrap(), c_to_str(pathname)); + return open_impl(cstring_to_str(&path), flags, || real!(openat)(dirfd, pathname, flags, filemode)); + } else { + ld_trace!("openat: Path {} is not a regular file, bypassing (dirfd={dirfd}, pathname={}).", path.to_str().unwrap(), c_to_str(pathname)); + } + } else { + ld_trace!("openat: Error resolving path {} from dirfd={dirfd} regular file, bypassing.", c_to_str(pathname)); + } + } else { + ld_trace!("openat: Interposing disabled; passing path={} from dirfd={dirfd} regular file, bypassing.", c_to_str(pathname)); + } + + real!(openat)(dirfd, pathname, flags, filemode) + } +} + +hook! { + unsafe fn read(fd: c_int, buf: *mut c_void, nbyte: size_t) -> ssize_t => my_read { + ld_func_trace!("read", fd, nbyte); + if interposing_disabled() || fd <= 2 { return real!(read)(fd, buf, nbyte); } + let _ig = with_interposing_disabled(); + + if let Some(fd_info) = maybe_fd_read_managed(fd) { + ld_trace!("read: Interposed read called with {nbyte} bytes on fd = {fd}"); + fd_info.read(buf, nbyte) + } else { + ld_trace!("read: non-interposed read called with {nbyte} bytes on fd = {fd}"); + real!(read)(fd, buf, nbyte) + } + } +} + +hook! { + unsafe fn fread(buf: *mut c_void, size: size_t, count: size_t, stream: *mut libc::FILE) -> size_t => my_fread { + ld_func_trace!("fread", size, count); + if interposing_disabled() { return real!(fread)(buf, size, count, stream); } + let _ig = with_interposing_disabled(); + + let fd = fileno(stream); + + if let Some(fd_info) = maybe_fd_read_managed(fd) { + ld_trace!("fread: Interposed read called with size={size}, count={count}, fd = {fd}"); + fd_info.fread(buf, size, count) + } else { + real!(fread)(buf, size, count, stream) + } + } +} + +unsafe fn stat_impl(fd: c_int, buf: *mut libc::stat) -> c_int { + let r = real!(fstat)(fd, buf); + + if r == EOF { + return EOF; + } + + if let Some(fd_info) = maybe_fd_read_managed(fd) { + ld_trace!("XetLDFS: fstat called on {fd} is managed"); + fd_info.update_stat(buf); + } + + r +} + +hook! { + unsafe fn fstat(fd: c_int, buf: *mut libc::stat) -> c_int => my_fstat { + ld_func_trace!("fstat", fd); + if interposing_disabled() { return real!(fstat)(fd, buf); } + let _ig = with_interposing_disabled(); + + stat_impl(fd, buf) + } +} + +unsafe fn real_fstat(fd: c_int, buf: *mut libc::stat) -> c_int { + real!(fstat)(fd, buf) +} + +#[cfg(target_os = "linux")] +hook! { + unsafe fn fstat64(fd: c_int, buf: *mut libc::stat) -> c_int => my_fstat64 { + ld_func_trace!("fstat64", fd); + if interposing_disabled() { return real!(fstat64)(fd, buf); } + let _ig = with_interposing_disabled(); + + stat_impl(fd, buf) + } +} + +hook! { + unsafe fn stat(pathname: *const libc::c_char, buf: *mut libc::stat) -> c_int => my_stat { + ld_func_trace!("stat", pathname); + if !process_in_interposable_state() { return real!(stat)(pathname, buf); } + + let fd = my_open(pathname, O_RDONLY, DEFFILEMODE); + if fd == -1 { + return real!(stat)(pathname, buf); + } + + stat_impl(fd, buf) + } +} + +#[allow(unused)] +unsafe fn real_stat(pathname: *const libc::c_char, buf: *mut libc::stat) -> c_int { + real!(stat)(pathname, buf) +} + +#[cfg(target_os = "linux")] +hook! { + unsafe fn stat64(pathname: *const libc::c_char, buf: *mut libc::stat) -> c_int => my_stat64 { + ld_func_trace!("stat64", pathname); + if !process_in_interposable_state() { return real!(stat64)(pathname, buf); } + + let fd = my_open(pathname, O_RDONLY, DEFFILEMODE); + if fd == -1 { + return real!(stat64)(pathname, buf); + } + + stat_impl(fd, buf) + } +} + +hook! { + unsafe fn fstatat(dirfd: libc::c_int, pathname: *const libc::c_char, buf: *mut libc::stat, flags: libc::c_int) -> libc::c_int => my_fstatat { + ld_func_trace!("fstatat", dirfd, pathname); + if !process_in_interposable_state() { return real!(fstatat)(dirfd, pathname, buf, flags); } + + let fd = { + if pathname.is_null() || *pathname == (0 as c_char) { + dirfd + } else { + my_openat(dirfd, pathname, flags, DEFFILEMODE) + } + }; + if fd == -1 { + return real!(fstatat)(dirfd, pathname, buf, flags); + } + + stat_impl(fd, buf) + } +} + +#[cfg(target_os = "linux")] +hook! { + unsafe fn fstatat64(dirfd: libc::c_int, pathname: *const libc::c_char, buf: *mut libc::stat, flags: libc::c_int) -> libc::c_int => my_fstatat64 { + ld_func_trace!("fstatat64", dirfd, pathname); + if !process_in_interposable_state() { return real!(fstatat)(dirfd, pathname, buf, flags); } + + let fd = { + if pathname.is_null() || *pathname == (0 as c_char) { + dirfd + } else { + my_openat(dirfd, pathname, flags, DEFFILEMODE) + } + }; + if fd == -1 { + return real!(fstatat64)(dirfd, pathname, buf, flags); + } + + stat_impl(fd, buf) + } +} + +#[cfg(target_os = "linux")] +hook! { + unsafe fn statx(dirfd: libc::c_int, pathname: *const libc::c_char, flags: libc::c_int, mask: libc::c_uint, statxbuf: *mut libc::statx) -> libc::c_int => my_statx { + ld_func_trace!("statx", dirfd, pathname, flags, mask); + if !process_in_interposable_state() { return real!(statx)(dirfd, pathname, flags, mask, statxbuf); } + + let fd = { + if pathname.is_null() || *pathname == (0 as c_char) { + dirfd + } else { + ld_trace!("statx: attempting to open path on {dirfd}, pathname = {:?}", c_to_str(pathname)); + my_openat(dirfd, pathname, flags, DEFFILEMODE) + } + }; + + if fd == -1 { + ld_trace!("statx: openat returned -1, passing through."); + return real!(statx)(dirfd, pathname, flags, mask, statxbuf); + } + + // If pathname is an empty string and the AT_EMPTY_PATH flag + // is specified in flags (see below), then the target file is + // the one referred to by the file descriptor dirfd. + let ret = real!(statx)(fd, C_EMPTY_STR, AT_EMPTY_PATH | flags, mask, statxbuf); + if ret == EOF { + return real!(statx)(dirfd, pathname, flags, mask, statxbuf); + } + + if let Some(fd_info) = maybe_fd_read_managed(fd) { + ld_trace!("statx: update_statx called on {fd}, is managed"); + fd_info.update_statx(statxbuf); + } else { + ld_trace!("statx called on {fd}; passed through."); + } + + + ret + } +} + +hook! { + unsafe fn lseek(fd: libc::c_int, offset: libc::off_t, whence: libc::c_int) -> libc::off_t => my_lseek { + ld_func_trace!("lseek", fd, offset, whence); + if interposing_disabled() { return real!(lseek)(fd, offset, whence); } + let _ig = with_interposing_disabled(); + + if let Some(fd_info) = maybe_fd_read_managed(fd) { + let ret = fd_info.lseek(offset, whence); + ld_trace!("XetLDFS: lseek called, offset={offset}, whence={whence}, fd={fd}: ret={ret}"); + ret + } else { + real!(lseek)(fd, offset, whence) + } + } +} + +hook! { + unsafe fn fseek(stream: *mut libc::FILE, offset: libc::c_long, whence: libc::c_int) -> libc::c_long => my_fseek { + ld_func_trace!("fseek", stream, offset, whence); + if interposing_disabled() || stream.is_null() { return real!(fseek)(stream, offset, whence); } + let _ig = with_interposing_disabled(); + + let fd = fileno(stream); + if fd < 0 { + return real!(fseek)(stream, offset, whence); + } + + if let Some(fd_info) = maybe_fd_read_managed(fd) { + let ret = fd_info.lseek(offset, whence) as libc::c_long; + ld_trace!("XetLDFS: lseek called, offset={offset}, whence={whence}, fd={fd}: ret={ret}"); + ret + } else { + real!(fseek)(stream, offset, whence) + } + } +} + +#[inline] +unsafe fn interposed_close(fd: libc::c_int) -> libc::c_int { + ld_trace!("close called on {fd}"); + close_fd_if_registered(fd); + real!(close)(fd) +} + +#[inline] +unsafe fn real_close(fd: libc::c_int) -> libc::c_int { + real!(close)(fd) +} + +hook! { + unsafe fn close(fd: libc::c_int) -> libc::c_int => my_close { + ld_func_trace!("close", fd); + if interposing_disabled() { return real!(close)(fd); } + let _ig = with_interposing_disabled(); + interposed_close(fd) + } +} + +hook! { + unsafe fn fclose(stream: *mut libc::FILE) -> libc::c_int => my_fclose { + ld_func_trace!("close", stream); + if interposing_disabled() || stream.is_null() { return real!(fclose)(stream); } + let _ig = with_interposing_disabled(); + + let fd = fileno(stream); + + ld_trace!("fclose called on {fd}"); + + close_fd_if_registered(fd); + + real!(fclose)(stream) + } +} + +hook! { + unsafe fn ftell(stream: *mut libc::FILE) -> libc::c_long => my_ftell { + ld_func_trace!("ftell", stream); + if interposing_disabled() || stream.is_null() { return real!(ftell)(stream); } + let _ig = with_interposing_disabled(); + + let fd = fileno(stream); + + if let Some(fd_info) = maybe_fd_read_managed(fd) { + let ret = fd_info.ftell() as libc::c_long; + ld_trace!("ftell: called on {fd}; interposed, ret = {ret}"); + ret + } else { + real!(ftell)(stream) + } + } +} + +hook! { + unsafe fn dup(old_fd: libc::c_int) -> libc::c_int => my_dup { + ld_func_trace!("dup", old_fd); + if interposing_disabled() { return real!(dup)(old_fd); } + let _ig = with_interposing_disabled(); + + let new_fd = real!(dup)(old_fd); + + if new_fd < 0 { + return new_fd; + } + + if let Some(fd_info) = maybe_fd_read_managed(old_fd) { + ld_trace!("dup: fd={new_fd} to point to same file as {old_fd}, path={:?}", fd_info.path()); + set_fd_read_interpose(new_fd, fd_info.dup(new_fd)); + } + + new_fd + } +} + +hook! { + unsafe fn dup2(old_fd: libc::c_int, new_fd: libc::c_int) -> libc::c_int => my_dup2 { + ld_func_trace!("dup2", old_fd, new_fd); + if interposing_disabled() { return real!(dup2)(old_fd, new_fd); } + let _ig = with_interposing_disabled(); + + let result = real!(dup2)(old_fd, new_fd); + if result < 0 { + return result; + } + + // If old_fd and new_fd are equal, then dup2() just returns new_fd; + // no other changes are made to the existing descriptor. + if old_fd != new_fd { + close_fd_if_registered(new_fd); + + if let Some(fd_info) = maybe_fd_read_managed(old_fd) { + ld_trace!("dup2: fd={new_fd} to point to same file as {old_fd}, path={:?}", fd_info.path()); + set_fd_read_interpose(new_fd, fd_info.dup(new_fd)); + } + } + + result + } +} + +#[cfg(target_os = "linux")] +hook! { + unsafe fn dup3(old_fd: libc::c_int, new_fd: libc::c_int, flags : libc::c_int) -> libc::c_int => my_dup3 { + if interposing_disabled() { return real!(dup3)(old_fd, new_fd, flags); } + let _ig = with_interposing_disabled(); + + let result = real!(dup3)(old_fd, new_fd, flags); + + if result == -1 { + return -1; + } + + close_fd_if_registered(new_fd); + + if let Some(fd_info) = maybe_fd_read_managed(old_fd) { + ld_trace!("dup3: fd={new_fd} to point to same file as {old_fd}, path={:?}", fd_info.path()); + set_fd_read_interpose(new_fd, fd_info.dup(new_fd)); + } + + result + } +} + +hook! { + unsafe fn mmap(addr: *mut libc::c_void, length: libc::size_t, prot: libc::c_int, flags: libc::c_int, fd: libc::c_int, offset: libc::off_t) -> *mut libc::c_void => my_mmap { + ld_func_trace!("mmap", length, prot, flags,fd, offset); + + // Avoid cyclic internal calls to malloc when actually allocating virtual memory, + // or when mmap non-regular file, e.g. shared memory + if (flags & libc::MAP_ANON != 0) || (flags & libc::MAP_ANONYMOUS != 0) || !is_regular_fd(fd) { + return real!(mmap)(addr, length, prot, flags, fd, offset); + } + + if process_in_interposable_state() { + if materialize_file_under_fd_if_needed(fd) { + ld_trace!("mmap: Materialized pointer file under descriptor {fd}."); + } else { + ld_trace!("mmap: fd={fd} not registered."); + } + } + + // Call the original mmap function + real!(mmap)(addr, length, prot, flags, fd, offset) + } +} diff --git a/xetldfs/src/path_utils.rs b/xetldfs/src/path_utils.rs new file mode 100644 index 00000000..49815520 --- /dev/null +++ b/xetldfs/src/path_utils.rs @@ -0,0 +1,148 @@ +use crate::{c_chars_to_cstring, real_fstat, real_stat}; +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; +use std::path::{Path, PathBuf}; + +pub fn resolve_path(raw_path: &str) -> Result { + let path = Path::new(raw_path); + + // Canonicalize the parent, which we expect to exist + if path.is_absolute() { + if let Some(parent) = path.parent() { + let canonical_parent = std::fs::canonicalize(parent)?; + Ok(canonical_parent.join(path.file_name().unwrap())) + } else { + Ok(path.to_path_buf()) + } + } else { + let abs_path = std::env::current_dir()?.join(path); + if let Some(parent) = abs_path.parent() { + let canonical_parent = std::fs::canonicalize(parent)?; + Ok(canonical_parent.join(abs_path.file_name().unwrap())) + } else { + Ok(abs_path) + } + } +} + +#[cfg(target_os = "linux")] +fn path_of_fd_impl(fd: libc::c_int) -> Option> { + let mut dest_path = vec![0 as c_char; libc::PATH_MAX as usize]; + + // On Linux, read the symbolic link at /proc/self/fd/dirfd + let path = format!("/proc/self/fd/{}\0", fd); + let c_path = CStr::from_bytes_with_nul(path.as_bytes()).unwrap(); + + let len = unsafe { + libc::readlink( + c_path.as_ptr(), + dest_path.as_mut_ptr() as *mut c_char, + dest_path.len(), + ) + }; + + if len < 0 { + return None; + } + + dest_path.truncate(len as usize); + Some(dest_path) +} + +#[cfg(target_os = "macos")] +fn path_of_fd_impl(fd: libc::c_int) -> Option> { + let mut dest_path = vec![0 as c_char; libc::PATH_MAX as usize]; + + // On macOS, use fcntl with F_GETPATH + if unsafe { libc::fcntl(fd, libc::F_GETPATH, dest_path.as_mut_ptr()) } == -1 { + return None; + } + + let len = dest_path + .iter() + .position(|&c| c == 0) + .unwrap_or(dest_path.len()); + + dest_path.truncate(len as usize); + Some(dest_path) +} + +pub fn path_of_fd(fd: libc::c_int) -> Option { + path_of_fd_impl(fd).map(c_chars_to_cstring) +} + +unsafe fn get_cwd() -> Option> { + let mut dest_path = vec![0 as c_char; libc::PATH_MAX as usize]; + + let cwd_ptr = unsafe { libc::getcwd(dest_path.as_mut_ptr(), dest_path.len()) }; + if cwd_ptr.is_null() { + return None; + } + + dest_path.truncate( + dest_path + .iter() + .position(|&c| c == 0) + .unwrap_or(dest_path.len()), + ); + Some(dest_path) +} + +pub fn resolve_path_from_fd(dirfd: libc::c_int, path: *const libc::c_char) -> Option { + unsafe { + if path.is_null() || *path == 0 { + let dest_path = path_of_fd_impl(dirfd)?; + return Some(c_chars_to_cstring(dest_path)); + } + + // Check if the path is absolute + if *path == b'/' as c_char { + return Some(CStr::from_ptr(path).to_owned()); + } + + let mut dest_path = { + if dirfd == libc::AT_FDCWD { + get_cwd()? + } else { + path_of_fd_impl(dirfd)? + } + }; + + assert_ne!(*dest_path.last().unwrap(), b'\0' as c_char); + dest_path.push(b'/' as c_char); + for i in 0.. { + let c = *(path.add(i)); + if c == 0 { + break; + } else { + dest_path.push(c); + } + } + + Some(c_chars_to_cstring(dest_path)) + } +} + +pub fn is_regular_file(pathname: *const libc::c_char) -> bool { + let mut buf: libc::stat = unsafe { std::mem::zeroed() }; + let buf_ptr = &mut buf as *mut libc::stat; + unsafe { + let ret = real_stat(pathname, buf_ptr); + if ret == -1 { + return false; + } + (*buf_ptr).st_mode & libc::S_IFMT == libc::S_IFREG + } +} + +pub fn is_regular_fd(fd: libc::c_int) -> bool { + let mut buf: libc::stat = unsafe { std::mem::zeroed() }; + let buf_ptr = &mut buf as *mut libc::stat; + unsafe { + let ret = real_fstat(fd, buf_ptr); + if ret == -1 { + return false; + } + (*buf_ptr).st_mode & libc::S_IFMT == libc::S_IFREG + } +} diff --git a/xetldfs/src/runtime.rs b/xetldfs/src/runtime.rs new file mode 100644 index 00000000..a010b5b7 --- /dev/null +++ b/xetldfs/src/runtime.rs @@ -0,0 +1,116 @@ +use lazy_static::lazy_static; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use tokio::runtime::{Builder, Runtime}; + +use crate::ld_trace; +use crate::ENABLE_CALL_TRACING; + +thread_local! { + static INTERPOSING_DISABLE_REQUESTS : AtomicU32 = const { AtomicU32::new(0) }; +} + +// Guaranteed to be zero on library load for all the static initializers. +// This will only be initialized once we register a file pointer for our own use. +lazy_static! { + static ref FD_RUNTIME_PID: u64 = unsafe { libc::getpid() as u64 }; +} + +static FD_RUNTIME_INITIALIZED: AtomicBool = AtomicBool::new(false); + +lazy_static! { + static ref TOKIO_RUNTIME: Runtime = { + let rt = Builder::new_multi_thread() + .worker_threads(1) + .on_thread_start(|| { + INTERPOSING_DISABLE_REQUESTS.with(|init| { + init.store(1, Ordering::Relaxed); + }); + }) + .enable_all() + .build() + .expect("Failed to create Tokio runtime"); + + rt + }; +} + +pub fn tokio_run(future: F) -> F::Output { + // This should never happen; is a problem. + assert!(runtime_activated()); + + use libc::{sigaction, sighandler_t, SIGCHLD, SIG_DFL}; + + // Save the current SIGCHLD signal handler so that bash doesn't interfere with the + // child processes. + let mut old_action: sigaction = unsafe { std::mem::zeroed() }; + if unsafe { libc::sigaction(SIGCHLD, std::ptr::null(), &mut old_action) } != 0 { + panic!("Failed to get the current SIGCHLD handler"); + } + + // Step 2: Set SIGCHLD to SIG_DFL (default action) + let mut new_action: sigaction = unsafe { std::mem::zeroed() }; + new_action.sa_sigaction = SIG_DFL as sighandler_t; + new_action.sa_flags = 0; + if unsafe { libc::sigaction(SIGCHLD, &new_action, std::ptr::null_mut()) } != 0 { + panic!("Failed to set SIGCHLD to default handler"); + } + + // Step 3: Run all the tokio stuff, which may include spawning other processes. + let result = tokio::task::block_in_place(|| TOKIO_RUNTIME.handle().block_on(future)); + + // Step 4: + if unsafe { libc::sigaction(SIGCHLD, &old_action, std::ptr::null_mut()) } != 0 { + panic!("Failed to restore the previous SIGCHLD handler"); + } + + result +} + +#[inline] +pub fn process_in_interposable_state() -> bool { + let pid = unsafe { libc::getpid() as u64 }; + let s_pid = *FD_RUNTIME_PID; + if pid == s_pid { + true + } else { + ld_trace!("XetLDFS: process not in interposable state: {pid} != {s_pid}"); + false + } +} + +pub fn activate_fd_runtime() { + FD_RUNTIME_INITIALIZED.store(true, Ordering::SeqCst); +} + +#[inline] +pub fn raw_runtime_activated() -> bool { + FD_RUNTIME_INITIALIZED.load(Ordering::Relaxed) +} + +#[inline] +pub fn runtime_activated() -> bool { + raw_runtime_activated() && process_in_interposable_state() +} + +#[inline] +pub fn interposing_disabled() -> bool { + if runtime_activated() { + INTERPOSING_DISABLE_REQUESTS.with(|init| init.load(Ordering::Relaxed) != 0) + } else { + true + } +} + +pub struct InterposingDisable {} + +impl Drop for InterposingDisable { + fn drop(&mut self) { + let v = INTERPOSING_DISABLE_REQUESTS.with(|v| v.fetch_sub(1, Ordering::Relaxed)); + assert_ne!(v, 0); + } +} + +pub fn with_interposing_disabled() -> InterposingDisable { + INTERPOSING_DISABLE_REQUESTS.with(|v| v.fetch_add(1, Ordering::Relaxed)); + InterposingDisable {} +} diff --git a/xetldfs/src/utils.rs b/xetldfs/src/utils.rs new file mode 100644 index 00000000..587a8273 --- /dev/null +++ b/xetldfs/src/utils.rs @@ -0,0 +1,104 @@ +use std::ffi::{CStr, CString}; +use std::io::ErrorKind; + +pub const C_EMPTY_STR: *const libc::c_char = c"".as_ptr() as *const libc::c_char; + +pub unsafe fn c_to_str<'a>(c_str: *const libc::c_char) -> &'a str { + if c_str.is_null() { + return ""; + } + + let c_str = CStr::from_ptr(c_str); + std::str::from_utf8_unchecked(c_str.to_bytes()) +} + +pub fn cstring_to_str(s: &CString) -> &str { + unsafe { std::str::from_utf8_unchecked(s.as_bytes()) } +} + +pub fn c_chars_to_cstring(mut vec: Vec) -> CString { + unsafe { + // A null terminator will be appended at the conversion below + while let Some(&0) = vec.last() { + vec.pop(); + } + + // Reinterpret the Vec as Vec without copying + let ptr = vec.as_mut_ptr() as *mut u8; + let len = vec.len(); + let cap = vec.capacity(); + + // Prevent the original vector from being dropped + std::mem::forget(vec); + + // Create the CString from the raw parts of the Vec + let u8_vec = Vec::from_raw_parts(ptr, len, cap); + CString::from_vec_unchecked(u8_vec) + } +} + +fn register_io_error_impl(err: std::io::Error, context: Option<&str>) -> std::io::Error { + use libc::*; + + let (err_code, err_msg) = match err.kind() { + ErrorKind::NotFound => (ENOENT, "File not found"), + ErrorKind::PermissionDenied => (EACCES, "Permission denied"), + ErrorKind::AlreadyExists => (EEXIST, "File already exists"), + ErrorKind::InvalidInput => (EINVAL, "Invalid input"), + ErrorKind::OutOfMemory => (ENOMEM, "Out of memory"), + ErrorKind::AddrInUse => (EADDRINUSE, "Address in use"), + ErrorKind::AddrNotAvailable => (EADDRNOTAVAIL, "Address not available"), + ErrorKind::BrokenPipe => (EPIPE, "Broken pipe"), + ErrorKind::ConnectionAborted => (ECONNRESET, "Connection aborted"), + ErrorKind::ConnectionRefused => (ECONNREFUSED, "Connection refused"), + ErrorKind::ConnectionReset => (ECONNRESET, "Connection reset"), + ErrorKind::Interrupted => (EINTR, "Interrupted"), + ErrorKind::InvalidData => (EINVAL, "Invalid data"), + ErrorKind::TimedOut => (ETIMEDOUT, "Operation timed out"), + ErrorKind::UnexpectedEof => (EIO, "Unexpected end of file"), + ErrorKind::WriteZero => (EIO, "Write zero"), + ErrorKind::WouldBlock => (EAGAIN, "Operation would block"), + ErrorKind::Unsupported => (ENOSYS, "Operation not supported"), + ErrorKind::Other => (EIO, "An unknown error occurred"), + _ => (EIO, "An unknown error occurred"), + }; + + errno::set_errno(errno::Errno(err_code)); + if let Some(ctx) = context { + eprintln!("XetFS Error: {err_msg}. {ctx}"); + } else { + eprintln!("XetFS Error: {err_msg}"); + } + err +} + +#[allow(dead_code)] +pub fn register_io_error(err: std::io::Error) -> std::io::Error { + register_io_error_impl(err, None) +} + +#[allow(dead_code)] +pub fn register_io_error_with_context(err: std::io::Error, context: &str) -> std::io::Error { + register_io_error_impl(err, Some(context)) +} + +pub fn open_flags_from_mode_string(mode: &str) -> Option { + use libc::{O_APPEND, O_CREAT, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY}; + + // File access mode flag "b" can optionally be specified to open a file in binary mode. + // This flag has no effect on POSIX systems but it's still valid. + let mode = mode.replace('b', ""); + + match mode.as_str() { + "r" => Some(O_RDONLY), + "r+" => Some(O_RDWR), + "w" => Some(O_WRONLY | O_CREAT | O_TRUNC), + "w+" => Some(O_RDWR | O_CREAT | O_TRUNC), + "a" => Some(O_WRONLY | O_CREAT | O_APPEND), + "a+" => Some(O_RDWR | O_CREAT | O_APPEND), + _ => { + eprintln!("XETLDFS Error: invalid mode string {mode}."); + None + } + } +} diff --git a/xetldfs/src/xet_interface.rs b/xetldfs/src/xet_interface.rs new file mode 100644 index 00000000..16f9a75a --- /dev/null +++ b/xetldfs/src/xet_interface.rs @@ -0,0 +1,268 @@ +use file_utils::SafeFileCreator; +use lazy_static::lazy_static; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, RwLock}; +use tokio::sync::Mutex as TMutex; + +use libxet::config::XetConfig; +use libxet::data::{PointerFile, PointerFileTranslatorV2}; +use libxet::errors::Result; +use libxet::git_integration::{get_repo_path, GitXetRepo}; +use libxet::ErrorPrinter; + +use crate::path_utils::{path_of_fd, resolve_path}; +use crate::runtime::{tokio_run, with_interposing_disabled}; +use crate::xet_rfile::XetFdReadHandle; +use crate::{cstring_to_str, real_close}; +use crate::{ld_trace, ld_warn, ENABLE_CALL_TRACING}; +use crate::{my_dup2, real_open}; + +lazy_static! { + static ref XET_REPO_WRAPPERS: RwLock>> = RwLock::new(Vec::new()); + static ref XET_ENVIRONMENT_CFG: TMutex> = TMutex::new(None); +} + +// Requires runnig inside tokio runtime, so async +async fn get_base_config() -> Result { + // If the base config isn't set, then initialize everthing. + let mut cfg_wrap = XET_ENVIRONMENT_CFG.lock().await; + + if cfg_wrap.is_none() { + let cfg = XetConfig::new(None, None, libxet::config::ConfigGitPathOption::NoPath)?; + + libxet::environment::log::initialize_tracing_subscriber(&cfg)?; + + let _ = openssl_probe::try_init_ssl_cert_env_vars(); + + ld_trace!("CFG initialized."); + + *cfg_wrap = Some(cfg); + } + + Ok(cfg_wrap.as_ref().unwrap().clone()) +} + +// Attempt to find all the instances. +pub fn get_repo_context(raw_path: &str) -> Result, PathBuf)>> { + let _ig = with_interposing_disabled(); + + ld_trace!("get_repo_context: {raw_path}"); + let path = resolve_path(raw_path).map_err(|e| { + ld_trace!("resolve_path failed: {e:?}"); + e + })?; + ld_trace!("get_repo_context: {raw_path} resolved to {path:?}"); + + if path + .components() + .any(|c| matches!(c, Component::Normal(name) if name == ".git")) + { + return Ok(None); + } + + ld_trace!("get_repo_context: {raw_path} is not inside .git"); + + // quick failure without trying opening **and implicitly setup** a repo. + let pf = PointerFile::init_from_path(&path); + ld_trace!("get_repo_context: pointer file is {pf:?}"); + if !pf.is_valid() { + ld_trace!("get_repo_context: {raw_path} is not a valid pointer file"); + return Ok(None); + } + + ld_trace!("get_repo_context: {raw_path} is a pointer file"); + + if let Some(repo_wrapper) = XET_REPO_WRAPPERS + .read() + .unwrap() + .iter() + .find(|xrw| path.starts_with(xrw.repo_path())) + .cloned() + { + ld_trace!("Xet instance found for {path:?} ( from {raw_path}"); + + return Ok(Some((repo_wrapper, path))); + } + + // See if we need to create it. + let Some(start_path) = path.parent() else { + return Ok(None); + }; + + // TODO: cache known directories as known non-xet paths. + let Some(repo_path) = get_repo_path(Some(start_path.to_path_buf())) + .map_err(|e| { + eprintln!("Error Initializing repo from {start_path:?} : {e:?}"); + e + }) + .unwrap_or(None) + else { + eprintln!("No repo path found for {start_path:?}"); + return Ok(None); + }; + + // TODO: Do more than print that we have this. + ld_trace!("Repo path for {path:?}: {repo_path:?}"); + + // Lock back here so we don't have multiple reads accessing the same repository + let mut xet_repo_wrappers = XET_REPO_WRAPPERS.write().unwrap(); + + // Check within the lock to make sure we're not opening multiple versions of this. + for xrw in xet_repo_wrappers.iter() { + if xrw.repo_path() == repo_path { + return Ok(Some((xrw.clone(), path))); + } + } + + let xet_repo = XetFSRepoWrapper::new(&repo_path) + .map_err(|e| { + ld_trace!("Error occurred initializing repo wrapper from {repo_path:?}: {e:?}"); + e + }) + .unwrap(); + + xet_repo_wrappers.push(xet_repo.clone()); + + Ok(Some((xet_repo, path))) +} + +pub struct XetFSRepoWrapper { + pub xet_repo: GitXetRepo, + pub pft: Arc, +} + +impl XetFSRepoWrapper { + pub fn new(root_path: impl AsRef) -> Result> { + let xrw = tokio_run(async move { + let base_cfg = get_base_config().await?; + + let cfg = base_cfg.switch_repo_path( + libxet::config::ConfigGitPathOption::PathDiscover(root_path.as_ref().to_path_buf()), + None, + )?; + + let xet_repo = GitXetRepo::open_and_verify_setup(cfg).await?; + let pft = Arc::new( + PointerFileTranslatorV2::from_config_smudge_only(&xet_repo.xet_config).await?, + ); + Result::Ok(Self { pft, xet_repo }) + })?; + + Ok(Arc::new(xrw)) + } + + pub fn repo_path(&self) -> &Path { + &self.xet_repo.repo_dir + } + + pub async fn open_path_for_read_if_pointer( + self: &Arc, + path: PathBuf, + ) -> Result> { + let pf = PointerFile::init_from_path(path); + + if !pf.is_valid() { + Ok(None) + } else { + Ok(Some(XetFdReadHandle::new(self.clone(), pf))) + } + } + + pub async fn materialize_path(&self, abs_path: impl AsRef) -> Result<()> { + let pf = PointerFile::init_from_path(&abs_path); + + let mut out_file = SafeFileCreator::replace_existing(&abs_path)?; + + self.pft + .smudge_file_from_pointer(abs_path.as_ref(), &pf, &mut out_file, None) + .await?; + + out_file.close()?; + + Ok(()) + } +} + +pub fn file_needs_materialization(open_flags: libc::c_int) -> bool { + let on = |flag| open_flags & flag != 0; + + let will_write = matches!(open_flags & libc::O_ACCMODE, libc::O_WRONLY | libc::O_RDWR); + + // need to materialize if writing and expect to keep any data + will_write && !on(libc::O_TRUNC) +} + +pub fn materialize_file_under_fd_if_needed(fd: libc::c_int) -> bool { + let _ig = with_interposing_disabled(); + + // Convert the file descriptor to a file path + if let Some(path) = path_of_fd(fd) { + ld_trace!("materialize_file_under_fd_if_needed: fd={fd}, path={path:?}"); + + // Materialize the file if it's a pointer file + if materialize_rw_file_if_needed(cstring_to_str(&path)) { + let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + if flags == -1 { + ld_warn!("materialize_file_under_fd: Error retrieving flags for orginial fd={fd}."); + return false; + } + + // Get the original file's mode + let file_mode = unsafe { libc::fcntl(fd, libc::F_GETFD) }; + if file_mode == -1 { + ld_warn!("materialize_file_under_fd: Error retrieving mode for orginial fd={fd}."); + return false; + } + + let new_fd = unsafe { real_open(path.as_ptr(), flags, file_mode as libc::mode_t) }; + + if new_fd == -1 { + ld_warn!( + "materialize_file_under_fd: Error opening materialized file at {path:?} : {:?}", + std::io::Error::last_os_error() + ); + return false; + } + + let dup2_res = unsafe { my_dup2(new_fd, fd) }; + + if dup2_res == -1 { + ld_warn!( + "materialize_file_under_fd: Error calling dup2 to replace old path: {:?}", + std::io::Error::last_os_error() + ); + return false; + } + + unsafe { real_close(new_fd) }; + + ld_trace!("materialize_file_under_fd_if_needed: fd={fd}, path={path:?} materialized."); + + return true; + } else { + ld_trace!( + "materialize_file_under_fd_if_needed: fd={fd}, path={path:?} not registered." + ); + } + } + false +} + +pub fn materialize_rw_file_if_needed(path: &str) -> bool { + ld_trace!("materialize_rw_file_if_needed: {path}"); + + if let Ok(Some((xet_repo, path))) = get_repo_context(path).map_err(|e| { + eprintln!("Error in get_repo_context for materializing {path}: {e:?}"); + e + }) { + tokio_run(async move { + let _ = xet_repo + .materialize_path(path) + .await + .log_error("Error Materializing path={path:?}"); + }); + true + } else { + false + } +} diff --git a/xetldfs/src/xet_rfile.rs b/xetldfs/src/xet_rfile.rs new file mode 100644 index 00000000..4a392d0a --- /dev/null +++ b/xetldfs/src/xet_rfile.rs @@ -0,0 +1,298 @@ +use crate::runtime::activate_fd_runtime; +use errno::{set_errno, Errno}; +use lazy_static::lazy_static; +use libc::*; +use libxet::data::PointerFile; +use libxet::errors::Result; +use libxet::ErrorPrinter; +use std::collections::HashMap; +use std::io::Cursor; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::sync::Arc; +use tokio::sync::Mutex as TMutex; + +use crate::ld_trace; +use crate::runtime; +use crate::ENABLE_CALL_TRACING; + +// size of buffer used by setbuf, copied from stdio.h +const BUFSIZ: c_int = 1024; + +// Copied from fread.c +// The maximum amount to read to avoid integer overflow. INT_MAX is odd, +// so it make sense to make it even. We subtract (BUFSIZ - 1) to get a +// whole number of BUFSIZ chunks. +const MAXREAD: c_int = c_int::MAX - (BUFSIZ - 1); + +use crate::xet_interface::{get_repo_context, XetFSRepoWrapper}; + +pub struct XetFdReadHandle { + xet_fsw: Arc, + pos: Arc>, + path: PathBuf, + fd: c_int, + pointer_file: Arc, // All non pointer files just get passed directly through +} + +lazy_static! { + static ref FD_LOOKUP: std::sync::RwLock>> = + std::sync::RwLock::new(HashMap::new()); +} + +pub fn set_fd_read_interpose(fd: c_int, fd_info: Arc) { + // We now have one thing to track, so go ahead and activate all the read and fstat commands. + activate_fd_runtime(); + + FD_LOOKUP.write().unwrap().insert(fd, fd_info); +} + +fn register_read_fd_impl(path: &str, fd: c_int) -> Result<()> { + ld_trace!("register_read_fd_impl: {path} for {fd}"); + if let Some((maybe_xet_wrapper, norm_path)) = get_repo_context(path)? { + ld_trace!("file norm_path: {norm_path:?} for {path}"); + + if let Some(mut fd_info) = runtime::tokio_run(async move { + maybe_xet_wrapper + .open_path_for_read_if_pointer(norm_path.clone()) + .await + .map_err(|e| { + ld_trace!("open for read if pointer failed: {e:?}"); + e + }) + .log_error(format!("Opening path {norm_path:?}.")) + })? { + fd_info.fd = fd; + + set_fd_read_interpose(fd, Arc::new(fd_info)); + + ld_trace!("{path} registered to {fd}"); + } else { + ld_trace!("open for read if pointer failed"); + } + } else { + ld_trace!("get repo context failed for : {path}"); + } + Ok(()) +} + +pub fn register_interposed_read_fd(path: &str, fd: c_int) { + // Possibly register the read fd. + let _ = register_read_fd_impl(path, fd).map_err(|e| { + ld_trace!("Error in register_read_fd with {path}: {e:?}"); + e + }); +} + +pub fn maybe_fd_read_managed(fd: c_int) -> Option> { + FD_LOOKUP.read().unwrap().get(&fd).cloned() +} + +pub fn close_fd_if_registered(fd: c_int) -> bool { + FD_LOOKUP.write().unwrap().remove_entry(&fd).is_some() +} + +impl XetFdReadHandle { + pub fn new(xet_fsw: Arc, pointer_file: PointerFile) -> Self { + Self { + xet_fsw, + pos: Arc::new(tokio::sync::Mutex::new(0)), + path: PathBuf::from_str(pointer_file.path()).unwrap(), + pointer_file: Arc::new(pointer_file), + fd: 0, + } + } + + pub fn filesize(&self) -> u64 { + self.pointer_file.filesize() + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn dup(self: Arc, new_fd: c_int) -> Arc { + Arc::new(Self { + xet_fsw: self.xet_fsw.clone(), + pos: self.pos.clone(), + path: self.path.clone(), + fd: new_fd, + pointer_file: self.pointer_file.clone(), + }) + } + + async fn read_impl(self: &Arc, buf: *mut c_void, n_bytes: size_t) -> ssize_t { + let slice = unsafe { std::slice::from_raw_parts_mut(buf as *mut u8, n_bytes) }; + + let mut out = Cursor::new(slice); + + let mut pos_lg = self.pos.lock().await; + let pos = *pos_lg; + + let end = (pos + n_bytes).min(self.pointer_file.filesize() as usize); + + let smudge_ok = self + .xet_fsw + .pft + .smudge_file_from_pointer(self.path(), &self.pointer_file, &mut out, Some((pos, end))) + .await + .log_error(format!( + "Smudging pointer file in range = ({pos},{end}); pointer file: \n{:?}", + &self.pointer_file + )) + .is_ok(); + + if smudge_ok { + *pos_lg = end; + (end - pos) as isize + } else { + 0 + } + } + + pub fn read(self: &Arc, buf: *mut c_void, n_bytes: size_t) -> ssize_t { + let s = self.clone(); + runtime::tokio_run(async move { s.read_impl(buf, n_bytes).await }) + } + + pub fn fread(self: &Arc, buf: *mut c_void, size: size_t, count: size_t) -> size_t { + let s = self.clone(); + + runtime::tokio_run(async move { + // adapted from fread.c + let mut resid = count * size; + + if resid == 0 { + return 0; + } + + let total = resid; + let mut ptr = buf; + + while resid > 0 { + let r: size_t = if resid > c_int::MAX as size_t { + MAXREAD as size_t + } else { + resid + }; + + let ret = s.read_impl(ptr, r).await; + + if ret == -1 { + // error occurred + todo!() + } + + let ret: size_t = ret.try_into().unwrap_or_default(); + + if ret != r { + return (total - resid + ret) / size; + } + + ptr = unsafe { ptr.byte_add(r) }; + resid -= r; + } + + // full read + count + }) + } + + pub fn update_stat(self: &Arc, buf: *mut libc::stat) { + unsafe { + (*buf).st_size = self.filesize() as i64; /* file size, in bytes */ + (*buf).st_blocks = 0; // todo!() /* blocks allocated for file */ + (*buf).st_blksize = libxet::merkledb::constants::IDEAL_CAS_BLOCK_SIZE + .try_into() + .unwrap() + /* optimal blocksize for I/O */ + } + } + + #[cfg(target_os = "linux")] + pub fn update_statx(self: &Arc, buf: *mut libc::statx) { + unsafe { + (*buf).stx_size = self.filesize(); /* file size, in bytes */ + (*buf).stx_blocks = 0; // todo!() /* blocks allocated for file */ + (*buf).stx_blksize = libxet::merkledb::constants::IDEAL_CAS_BLOCK_SIZE + .try_into() + .unwrap() + /* optimal blocksize for I/O */ + } + } + + pub fn lseek(self: &Arc, offset: libc::off_t, whence: libc::c_int) -> libc::off_t { + let s = self.clone(); + + runtime::tokio_run(async move { + // whence is not valid? + if !matches!( + whence, + SEEK_SET | SEEK_CUR | SEEK_END | SEEK_DATA | SEEK_HOLE + ) { + set_errno(Errno(libc::EINVAL)); + return EOF.into(); + } + + let fsize = s.pointer_file.filesize(); + + // lock because it's difficult to implement with pure atomic variable. + let mut pos_lock = s.pos.lock().await; + + let cur_pos = *pos_lock as u64; + + // The seek location (calculated from offset and whence) is negative? + let seek_to_negtive_location = match whence { + SEEK_SET => offset.is_negative(), + SEEK_CUR => offset.is_negative() && cur_pos < offset.abs().try_into().unwrap(), + SEEK_END => offset.is_negative() && fsize < offset.abs().try_into().unwrap(), + _ => false, // noop + }; + + if seek_to_negtive_location { + set_errno(Errno(libc::EINVAL)); + return EOF.into(); + } + + // The seek location is too large to be stored in an object of type off_t? + let seek_overflow = match whence { + SEEK_SET => false, + SEEK_CUR => libc::off_t::MAX.saturating_sub_unsigned(cur_pos) < offset, + SEEK_END => libc::off_t::MAX.saturating_sub_unsigned(fsize) < offset, + _ => false, // noop + }; + + if seek_overflow { + set_errno(Errno(libc::EOVERFLOW)); + return EOF.into(); + } + + // whence is SEEK_DATA or SEEK_HOLE, and offset is beyond the end of the file? + if !matches!(whence, SEEK_DATA | SEEK_HOLE) + && offset.is_positive() + && offset as u64 == fsize + { + set_errno(Errno(libc::ENXIO)); + return EOF.into(); + } + + let new_pos = match whence { + SEEK_SET => offset.try_into().unwrap(), + SEEK_CUR => cur_pos.saturating_add_signed(offset), + SEEK_END => fsize.saturating_add_signed(offset), + SEEK_DATA => offset.try_into().unwrap(), // always data + SEEK_HOLE => fsize, // no hole + _ => unreachable!(), + }; + + *pos_lock = new_pos as usize; + + new_pos as libc::off_t + }) + } + + pub fn ftell(self: &Arc) -> libc::c_long { + let s = self.clone(); + runtime::tokio_run(async move { *s.pos.lock().await as libc::c_long }) + } +} diff --git a/xetldfs/tests/integration_tests.rs b/xetldfs/tests/integration_tests.rs new file mode 100644 index 00000000..d0d2333d --- /dev/null +++ b/xetldfs/tests/integration_tests.rs @@ -0,0 +1,166 @@ +use anyhow::anyhow; +use std::{io::Write, path::Path, process::Command}; +use tempfile::TempDir; +use tracing::info; + +/// Set this to true to see the output of the tests on success. +const DEBUG: bool = false; + +struct IntegrationTest { + test_script: String, + arguments: Vec, + assets: Vec<(String, &'static [u8])>, +} + +impl IntegrationTest { + fn new(test_script: &str) -> Self { + Self { + test_script: test_script.to_owned(), + arguments: Vec::new(), + assets: Vec::new(), + } + } + + #[allow(unused)] + fn add_arguments(&mut self, args: &[&str]) { + self.arguments.extend(args.iter().map(|s| s.to_string())) + } + + #[allow(unused)] + fn add_asset(&mut self, name: &str, arg: &'static [u8]) { + self.assets.push((name.to_owned(), arg)); + } + + fn run(&self) -> anyhow::Result<()> { + // Create a temporary directory + let tmp_repo_dest = TempDir::new().unwrap(); + let tmp_path_path = tmp_repo_dest.path().to_path_buf(); + + std::fs::write(tmp_path_path.join("test_script.sh"), &self.test_script).unwrap(); + + std::fs::write( + tmp_path_path.join("initialize.sh"), + include_str!("integration_tests/initialize.sh"), + ) + .unwrap(); + + std::fs::write( + tmp_path_path.join("setup_test_repo.sh"), + include_str!("integration_tests/setup_test_repo.sh"), + ) + .unwrap(); + + // Write the assets into the tmp path + for (name, data) in self.assets.iter() { + std::fs::write(tmp_path_path.join(name), data)?; + } + + let mut cmd = Command::new("bash"); + cmd.args(["-e", "-x", "test_script.sh"]); + cmd.args(&self.arguments[..]); + cmd.current_dir(tmp_path_path.clone()); + // Add in the path of the git-xet / xetcat executable + + let git_xet_path = env!("CARGO_BIN_EXE_git-xet"); + let buildpath = Path::new(&git_xet_path).parent().unwrap(); + info!("Adding {:?} to path.", &buildpath); + cmd.env( + "PATH", + format!( + "{}:{}", + &buildpath.to_str().unwrap(), + &std::env::var("PATH").unwrap() + ), + ); + + // Export the path of the ld_preload lib + let lib_name = env!("CARGO_PKG_NAME"); + let lib_file = if cfg!(target_os = "linux") { + format!("lib{}.so", lib_name) + } else if cfg!(target_os = "macos") { + format!("lib{}.dylib", lib_name) + } else { + panic!("Unsupported target OS"); + }; + + cmd.env("LDPRELOAD_LIB", buildpath.join(lib_file).as_os_str()); + + // Now, to prevent ~/.gitconfig to be read, we need to reset the home directory; otherwise + // these tests will not be run in an isolated environment. + // + // NOTE: this is not a problem with git version 2.32 or later. There, GIT_CONFIG_GLOBAL + // works and the scripts take advantage of it. However, outside of that, this is needed + // to avoid issues with a lesser git. + cmd.env("HOME", tmp_path_path.as_os_str()); + + // Set defaults for all of these, but allow them to be overridden + cmd.env( + "XET_LOG_LEVEL", + std::env::var_os("XET_LOG_LEVEL").unwrap_or("error".into()), + ); + cmd.env( + "XET_GLOBAL_DEDUP_POLICY", + std::env::var_os("XET_GLOBAL_DEDUP_POLICY").unwrap_or("always".into()), + ); + cmd.env( + "XET_SHARD_QUERY_POLICY", + std::env::var_os("XET_SHARD_QUERY_POLICY").unwrap_or("LocalFirst".into()), + ); + + cmd.env("XET_AXE_ENABLED", "false"); + + // Now, run the script. + let out = cmd.output()?; + let status = out.status; + + if status.success() { + if DEBUG { + // Just dump things to the output + eprintln!("Test succeeded, STDOUT:"); + std::io::stdout().write_all(&out.stdout).unwrap(); + eprintln!("STDERR:"); + std::io::stderr().write_all(&out.stderr).unwrap(); + } + Ok(()) + } else { + eprintln!("Test failed, STDOUT:"); + std::io::stderr().write_all(&out.stderr).unwrap(); + // Parse output for error string: + let stderr_out = std::str::from_utf8(&out.stderr)?; + + eprintln!("STDERR:\n{}", &stderr_out); + + let error_re = regex::Regex::new("ERROR:>>>>>(.*)<<<<<").unwrap(); + + let captures = error_re.captures(stderr_out); + + if let Some(captured_text) = captures { + Err(anyhow!( + "Test failed: {}", + captured_text.get(1).unwrap().as_str() + )) + } else { + Err(anyhow!("Test failed: Unknown Error.")) + } + } + } +} + +#[cfg(all(test, unix))] +mod git_integration_tests { + use super::*; + #[test] + fn test_basic_read() -> anyhow::Result<()> { + IntegrationTest::new(include_str!("integration_tests/test_basic_read.sh")).run() + } + + #[test] + fn test_basic_write() -> anyhow::Result<()> { + IntegrationTest::new(include_str!("integration_tests/test_basic_write.sh")).run() + } + + #[test] + fn test_bulk_write() -> anyhow::Result<()> { + IntegrationTest::new(include_str!("integration_tests/test_bulk_write.sh")).run() + } +} diff --git a/xetldfs/tests/integration_tests/initialize.sh b/xetldfs/tests/integration_tests/initialize.sh new file mode 100644 index 00000000..c3a9c2f4 --- /dev/null +++ b/xetldfs/tests/integration_tests/initialize.sh @@ -0,0 +1,434 @@ +#!/usr/bin/env bash + +export XET_LOG_FORMAT=compact +export XET_DISABLE_VERSION_CHECK="1" + +# Workaround for git reference transaction hook issues +export GIT_CLONE_PROTECTION_ACTIVE=false + +# With these, Log the filename, function name, and line number when showing where we're executing. +set -o xtrace +export PS4='+($(basename ${BASH_SOURCE}):${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +if [[ "$OSTYPE" == "darwin"* ]] ; then + echo "OSX ARCH: Current archetecture is: $(arch)" +fi + +setup_isolated_git_environment() { + # Set up local, self-contained config stuff to make sure the environment for the tests is hermetic. + export GIT_CONFIG_GLOBAL="$PWD/.gitconfig" + + + if [[ ! -e $GIT_CONFIG_GLOBAL ]]; then + echo "[user] + name = Xet Tester + email = test@xetdata.com + " >$GIT_CONFIG_GLOBAL + else + die "These tests may overwrite global settings; please run \ + them in a directory without a .gitconfig present." + fi + + # Do some checks to make sure that everything is set up + username=$(git config --get user.name || echo "") + [[ ! -z $username ]] || die "Git config user.name not set." + + useremail=$(git config --get user.email || echo "") + [[ ! -z $useremail ]] || die "Git config user.email not set." + + git config --global init.defaultBranch main + git config --global --unset-all filter.xet.process || echo "global already unset" + git config --global --unset-all filter.xet.required || echo "global already unset" + + git xet install +} + +setup_local_xet_environment() { + if [[ -z $XET_TESTING_REMOTE ]]; then + if [[ -z $XET_CAS_SERVER ]]; then + # In Cygwin or msys emulators, $PWD is returned in unix format. Directly + # exporting XET_CAS_SERVER using this path format will crash git-xet because + # a Windows build cannot understand such a path. + # We convert it to Windows format using cygpath. + local pwd=$PWD + if [[ "$OSTYPE" == "cygwin" || "$OSTYPE" == "msys" ]]; then + pwd=$(cygpath -wa $pwd) + fi + export XET_CAS_SERVER="local://$pwd/cas" + mkdir -p "$PWD/cas" + fi + fi + + # Set up logging + export XET_PRINT_LOG_FILE_PATH=1 + export XET_LOG_PATH="$PWD/logs/log_{timestamp}_{pid}.txt" +} + +setup_testing_environment() { + setup_isolated_git_environment + setup_local_xet_environment + + # This is needed as older versions of git only go to $HOME/.gitconfig and do not respect + # the GIT_CONFIG_GLOBAL environment variable. + export HOME=$PWD + export base_dir="$HOME" +} + +# Sets up a local testing environment in a specific directory. +setup_xetldfs_testing_env() { + + if [[ -z "$LDPRELOAD_LIB" ]] ; then + + if [[ ! -z $(which x) ]] ; then + target_path="$(dirname $(which x))" + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + if [[ -e "${target_path}/libxetldfs.so" ]] ; then + export LDPRELOAD_LIB=${target_path}/libxetldfs.so + fi + elif [[ "$OSTYPE" == "darwin"* ]]; then + if [[ -e "${target_path}/libxetldfs.dylib" ]] ; then + export LDPRELOAD_LIB=${target_path}/libxetldfs.dylib + fi + fi + fi + fi + + if [[ -z "$LDPRELOAD_LIB" ]] ; then + die "Set LDPRELOAD_LIB to the location of the dylib file." + fi + + if [[ ! -e "$LDPRELOAD_LIB" ]] ; then + die "Dylib $LDPRELOAD_LIB does not exist." + fi + + export PATH="$(dirname $LDPRELOAD_LIB):$PATH" + + if [[ ! -e $(which x) ]] ; then + die "Cannot find x binary along with $LDPRELOAD_LIB" + fi + + if [[ $(x verify) != "VERIFICATION" ]] ; then + die "Wrong x binary?" + fi + + setup_testing_environment + setup_xetldfs "$LDPRELOAD_LIB" +} + +setup_test_repos() { + # file larger than 100 bytes will be checked-in as pointer file + export XET_CAS_SIZETHRESHOLD=100 + + remote=$(create_bare_repo) + git clone $remote repo_1 + + pushd repo_1 + + [[ $(git branch) == *"main"* ]] || git checkout -b main + git xet init --force + git push origin main # This created a commit, so push it to main. + + create_text_file text_data.txt key1 1000 + git add text_data.txt + git commit -a -m "Test file." + git push origin main + + popd + + git xet clone --lazy $remote repo_xetldfs +} + +# Use. After running this. +setup_xetldfs() { + local PS4="" + export XETLD_LIB=$(realpath $1) + + >&2 echo "Using $XETLD_LIB for absolute path." + + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + export x_cat=cat + export x_stat=stat + elif [[ "$OSTYPE" == "darwin"* ]]; then + export x_cat="x cat" + export x_stat="x stat" + else + echo "Unable to set up local environment." + return 1 + fi +} + +xetfs_on() { + export LD_PRELOAD=$XETLD_LIB + export DYLD_INSERT_LIBRARIES=$XETLD_LIB +} + +xetfs_off() { + unset LD_PRELOAD + unset DYLD_INSERT_LIBRARIES +} + +die() { + echo >&2 "ERROR:>>>>> $1 <<<<<" + return 1 +} +export -f die + +# support both Mac OS and Linux for these scripts +if hash md5 2>/dev/null; then + checksum() { + md5 -q $1 + } + checksum_string() { + echo $1 | md5 -q + } +else + checksum() { + md5sum $1 | head -c 32 + } + checksum_string() { + echo $1 | md5sum | head -c 32 + } +fi + + +export -f checksum +export -f checksum_string + +create_bare_repo() { + # Clean up the remote repo. + if [[ ! -z $XET_TESTING_REMOTE ]]; then + # Reset the remote branch main to a single initial commit + rm >&2 -rf origin_blanch && mkdir origin_blank && cd origin_blank + git >&2 init + git >&2 xet init --local --force + git >&2 remote add origin $XET_TESTING_REMOTE + git >&2 fetch --all + git >&2 push --force origin main + + # Delete all other branches + remotes_to_del=$(git branch -r -l --format '%(refname)' | sed 's|refs/remotes/origin/||' | grep -v HEAD | grep -v main | grep -v notes) >&2 + + for branch in $remotes_to_del; do + echo >&2 "Deleting remote branch $branch on remote." + git >&2 push origin --delete $branch + done + + git >&2 clone $XET_TESTING_REMOTE origin_tmp + pushd >&2 origin_tmp + + popd >&2 + echo $XET_TESTING_REMOTE + else + repo=origin >&2 + rm >&2 -rf $repo + mkdir >&2 -p $repo + pushd >&2 $repo + git >&2 init --bare --initial-branch=main + popd >&2 + + echo $PWD/$repo + fi +} +export -f create_bare_repo + +create_bare_xet_repo() { + # Clean up the remote repo. + if [[ ! -z $XET_TESTING_REMOTE ]]; then + create_bare_repo $@ + else + repo=origin >&2 + rm >&2 -rf $repo + mkdir >&2 -p $repo + pushd >&2 $repo + git >&2 init --bare --initial-branch=main + git >&2 xet init + popd >&2 + + echo $PWD/$repo + fi +} +export -f create_bare_xet_repo + +create_data_file() { + f="$1" + len=$2 + + printf '\xff' >$f # Start with this to ensure utf-8 encoding fails quickly. + cat /dev/random | head -c $(($2 - 1)) >>$f + echo $(checksum $f) +} +export -f create_data_file + +append_data_file() { + f="$1" + len=$2 + + printf '\xff' >>$f # Start with this to ensure utf-8 encoding fails quickly. + cat /dev/random | head -c $(($2 - 1)) >>$f + echo $(checksum $f) +} +export -f append_data_file + +assert_files_equal() { + # Use fastest way to determine content equality. + cmp --silent $1 $2 || die "Assert Failed: Files $1 and $2 not equal." +} +export -f assert_files_equal + +assert_files_not_equal() { + # Use fastest way to determine content equality. + cmp --silent $1 $2 && die "Assert Failed: Files $1 and $2 should not be equal." || echo >&2 "Files $1 and $2 not equal." +} +export -f assert_files_not_equal + +assert_stored_as_pointer_file() { + local LD_PRELOAD="" + local DYLD_INSERT_LIBRARIES="" + file=$1 + match=$(git show HEAD:$file | head -n 1 | grep -F '# xet version' || echo "") + [[ ! -z "$match" ]] || die "File $file does not appear to be stored as a pointer file." +} +export -f assert_stored_as_pointer_file + +assert_stored_as_full_file() { + local LD_PRELOAD="" + local DYLD_INSERT_LIBRARIES="" + file=$1 + match=$(git show HEAD:$file | head -n 1 | grep -F '# xet version' || echo "") + [[ -z "$match" ]] || die "File $file does not appear to be stored as a pointer file." +} +export -f assert_stored_as_full_file + +assert_is_pointer_file() { + local LD_PRELOAD="" + local DYLD_INSERT_LIBRARIES="" + file=$1 + match=$(cat $file | head -n 1 | grep -F '# xet version' || echo "") + [[ ! -z "$match" ]] || die "File $file does not appear to be a pointer file." +} +export -f assert_is_pointer_file + +assert_pointer_file_size() { + local LD_PRELOAD="" + local DYLD_INSERT_LIBRARIES="" + file=$1 + size=$2 + + assert_is_pointer_file $file + + filesize=$(cat $file | grep -F filesize | sed -E 's|.*filesize = ([0-9]+).*|\1|' || echo "") + [[ $filesize == $size ]] || die "Pointer file $file gives incorrect size; $filesize, expected $size." +} +export -f assert_pointer_file_size + +pseudorandom_stream() { + key=$1 + + while true; do + key=$(checksum_string $key) + echo "$(echo $key | xxd -r -p)" 2>/dev/null || exit 0 + done +} +export -f pseudorandom_stream + +create_csv_file() { + local LD_PRELOAD="" + local DYLD_INSERT_LIBRARIES="" + local set_x_status=$([[ "$-" == *x* ]] && echo 1) + set +x + + csv_file="$1" + key="$2" + n_lines="$3" + n_repeats="${4:-1}" + n_lines_p_1=$((n_lines + 1)) + + pseudorandom_stream "$key" | hexdump -v -e '5/1 "%02x""\n"' | + awk -v OFS='\t' 'NR == 1 { print "foo", "bar", "baz" } + { print "S"substr($0, 1, 4), substr($0, 5, 2), substr($0, 7, 2)"."substr($0, 9, 1), 6, 3}' | + head -n $((n_lines + 1)) | tr 'abcdef' '123456' >$csv_file.part + + cat $csv_file.part >$csv_file + + for i in {0..n_repeats}; do + tail -n $n_lines $csv_file.part >>$csv_file + done + + rm $csv_file.part + [[ $set_x_status != "1" ]] || set -x +} +export -f create_csv_file + +create_random_csv_file() { + f="$1" + n_lines="$2" + n_repeats="${3:-1}" + n_lines_p_1=$((n_lines + 1)) + + cat /dev/random | hexdump -v -e '5/1 "%02x""\n"' | + awk -v OFS='\t' 'NR == 1 { print "foo", "bar", "baz" } + { print "S"substr($0, 1, 4), substr($0, 5, 2), substr($0, 7, 2)"."substr($0, 9, 1), 6, 3}' | + head -n $((n_lines + 1)) | tr 'abcdef' '123456' >$f.part + + cat $f.part >$f + + for i in {0..n_repeats}; do + tail -n $n_lines $f.part >>$f + done + + rm $f.part +} +export -f create_random_csv_file + +create_text_file() { + local LD_PRELOAD="" + local DYLD_INSERT_LIBRARIES="" + local set_x_status=$([[ "$-" == *x* ]] && echo 1) + set +x + + text_file="$1" + key="$2" + n_lines="$3" + n_repeats="${4:-1}" + + create_csv_file "$text_file.temp" "$key" "$n_lines" "$n_repeats" + + cat "$text_file.temp" | tr ',0123456789' 'ghijklmnopq' >$text_file + rm "$text_file.temp" + [[ $set_x_status != "1" ]] || set -x +} +export -f create_text_file + +random_tag() { + cat /dev/random | head -c 64 | checksum_string +} +export -f random_tag + +file_size() { + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + stat --printf="%s" $1 + elif [[ "$OSTYPE" == "darwin"* ]]; then + x stat $1 + fi +} + +raw_file_size() { + local LD_PRELOAD="" + local DYLD_INSERT_LIBRARIES="" + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + stat --printf="%s" $1 + elif [[ "$OSTYPE" == "darwin"* ]]; then + stat $1 + fi +} + + +assert_file_size() { + len=$(file_size $1) + [[ $len == $2 ]] || die "Size of file $1 is $len, expected = $2" +} + +assert_raw_file_size() { + len=$(raw_file_size $1) + [[ $len == $2 ]] || die "Size of file $1 is $len, expected = $2" +} diff --git a/xetldfs/tests/integration_tests/setup_test_repo.sh b/xetldfs/tests/integration_tests/setup_test_repo.sh new file mode 100644 index 00000000..830cad02 --- /dev/null +++ b/xetldfs/tests/integration_tests/setup_test_repo.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Sourcing this script, after initialize.sh, gives us $remote, $text_1, $text_1_len, $text_2, +# $text_2_len, $all_file_text, etc. +# +# It also sets XET_CAS_SIZETHRESHOLD such that $text_1 is stored in CAS and as a pointer file. +# +# Repo from in $remote has 12 files, t1.txt, t2.txt, t3.txt, t4.txt, and same for l and m instead of t. +# All are stored as pointer files and resolve in content to $text_1. +# +# Calling verify_size checks the size of things too. + +setup_xetldfs_testing_env + +git xet install + +remote=$(create_bare_repo) + +git clone $remote repo_setup + +# file larger than 100 bytes will be checked-in as pointer file +export XET_CAS_SIZETHRESHOLD=16 +text_1="abcdefghijklmnopqrstuvwxyz" +text_2="0123456789" +text_ins_at_10="${text_1:0:10}${text_2}${text_1:20}" + +text_1_len=$(echo -n $text_1 | wc -c) +text_2_len=$(echo -n $text_2 | wc -c) +text_12_len=$(echo -n "${text_1}${text_2}" | wc -c) + + +pushd repo_setup +[[ $(git branch) == *"main"* ]] || git checkout -b main +git xet init --force +git push origin main # This created a commit, so push it to main. + +echo -n $text_1 > text_data.txt +text_data_file_size=$(file_size text_data.txt) + +for n in 1 2 3 4 ; do + cp text_data.txt t$n.txt + cp text_data.txt m$n.txt + cp text_data.txt l$n.txt +done + +all_file_text="$(cat t?.txt)" + +git add . +git commit -m "add text data" +git push origin main +popd + +# Some helper functions + +verify_size() { + file=$1 + expected_len=$2 + ( + xetfs_on + len=$(x stat $file) + [[ $len == $expected_len ]] || die "x stat length of $file is wrong; got $len, expected $expected_len" + + len=$(x fstat $file) + [[ $len == $expected_len ]] || die "x fstat length of $file is wrong; got $len, expected $expected_len" + + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + len=$(stat --printf="%s" $file) + [[ $len == $expected_len ]] || die "linux stat length of $file is wrong; got $len, expected $expected_len" + fi + ) +} diff --git a/xetldfs/tests/integration_tests/test_basic_read.sh b/xetldfs/tests/integration_tests/test_basic_read.sh new file mode 100644 index 00000000..7bed43f9 --- /dev/null +++ b/xetldfs/tests/integration_tests/test_basic_read.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -e +set -x + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" &>/dev/null && pwd 2>/dev/null)" +. "$SCRIPT_DIR/initialize.sh" + +# This gives us $remote, $text_1, $text_1_len, $text_2, $text_2_len, $all_file_text, etc. +# It also sets XET_CAS_SIZETHRESHOLD such that $text_1 is stored in CAS and as a pointer file. +# +# Repo from in $remote has 12 files, t1.txt, t2.txt, t3.txt, t4.txt, and same for l and m instead of t. +# All are stored as pointer files and resolve in content to $text_1. +. "$SCRIPT_DIR/setup_test_repo.sh" + +git xet clone --lazy $remote repo + +pushd repo +assert_is_pointer_file text_data.txt + +for n in 1 2 3 4 ; do + assert_is_pointer_file t$n.txt + verify_size t$n.txt $text_1_len + + assert_is_pointer_file m$n.txt + verify_size m$n.txt $text_1_len + + assert_is_pointer_file l$n.txt + verify_size l$n.txt $text_1_len +done + +# Read +( + xetfs_on + [[ "$(x cat t1.txt)" == "$text_1" ]] || die "t1.txt not read as pointer." + [[ "$(x cat t?.txt)" == "$all_file_text" ]] || die "all text does not match correctly." + [[ "$(x cat-mmap m1.txt)" == "$text_1" ]] || die "m1.txt not read through mmap." + [[ "$(x cat-mmap m?.txt)" == "$all_file_text" ]] || die "all text does not match correctly with mmap." + + # With linux + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + [[ "$(bash -c 'x cat t1.txt')" == "$text_1" ]] || die "t1.txt not read as pointer." + [[ "$(bash -c 'x cat t?.txt')" == "$all_file_text" ]] || die "all text does not match correctly." + [[ "$(bash -c 'x cat-mmap m1.txt')" == "$text_1" ]] || die "m1.txt not read through mmap." + [[ "$(bash -c 'x cat-mmap m?.txt')" == "$all_file_text" ]] || die "all text does not match correctly with mmap." + + [[ "$(cat l1.txt)" == "$text_1" ]] || die "l1.txt not read as pointer." + [[ "$(cat l*.txt)" == "$all_file_text" ]] || die "all text does not match correctly." + [[ "$(bash -c 'cat l1.txt')" == "$text_1" ]] || die "m1.txt not read through bash cat." + [[ "$(bash -c 'cat l*.txt')" == "$all_file_text" ]] || die "all text does not match correctly in linux bash." + [[ "$(bash -c 'x cat-mmap l1.txt')" == "$text_1" ]] || die "l1.txt not read through bash cat." + [[ "$(bash -c 'x cat-mmap l?.txt')" == "$all_file_text" ]] || die "all text does not match correctly in linux bash." + + # Ensure that things work when the library is loaded in bash. + [[ "$(bash -c 'cat t1.txt > /dev/null ; x cat t1.txt')" == "$text_1" ]] || die "t1.txt not read as pointer." + [[ "$(bash -c 'cat t?.txt > /dev/null ; x cat t?.txt')" == "$all_file_text" ]] || die "all text does not match correctly." + [[ "$(bash -c 'cat m1.txt > /dev/null ; x cat-mmap m1.txt')" == "$text_1" ]] || die "m1.txt not read through mmap." + [[ "$(bash -c 'cat m?.txt > /dev/null ; x cat-mmap m?.txt')" == "$all_file_text" ]] || die "all text does not match correctly with mmap." + fi +) + +popd \ No newline at end of file diff --git a/xetldfs/tests/integration_tests/test_basic_write.sh b/xetldfs/tests/integration_tests/test_basic_write.sh new file mode 100644 index 00000000..585951cc --- /dev/null +++ b/xetldfs/tests/integration_tests/test_basic_write.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +set -e +set -x + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" &>/dev/null && pwd 2>/dev/null)" +. "$SCRIPT_DIR/initialize.sh" + +# This gives us $remote, $text_1, $text_1_len, $text_2, $text_2_len, $all_file_text, etc. +# It also sets XET_CAS_SIZETHRESHOLD such that $text_1 is stored in CAS and as a pointer file. +# +# Repo from in $remote has 12 files, t1.txt, t2.txt, t3.txt, t4.txt, and same for l and m instead of t. +# All are stored as pointer files and resolve in content to $text_1. +. "$SCRIPT_DIR/setup_test_repo.sh" + +git xet clone --lazy $remote repo_3 +pushd repo_3 + +for n in 1 2 3 4 ; do + assert_is_pointer_file t$n.txt + assert_is_pointer_file m$n.txt + assert_is_pointer_file l$n.txt +done + +( + xetfs_on + [[ $(x cat t2.txt) == $text_1 ]] || die "Issue getting t2.txt." + + verify_size t2.txt $text_1_len + echo -n $text_2 | x write t2.txt + + [[ $(x cat t2.txt) == $text_2 ]] || die "rewrite pointer file failed" + + verify_size t2.txt $text_2_len + + # Overwrite mmap + echo -n $text_2 | x write-mmap m2.txt + [[ "$(x cat m2.txt)" == "$text_2" ]] || die "m2.txt not overwritten." + verify_size m2.txt $text_2_len + + # Overwrite, linux specific + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + # ">" is a bash feature and exporting LD_PRELOAD in xetfs_on doesn't + # affect the current bash process + bash -c "echo -n $text_2 > l2.txt" + [[ "$(cat l2.txt)" == "$text_2" ]] || die "l2.txt not overwritten." + fi +) + +# Append +assert_is_pointer_file t3.txt +assert_is_pointer_file m3.txt +assert_is_pointer_file l3.txt + +( + xetfs_on + # Regular + echo -n $text_2 | x append t3.txt + [[ $(cat t3.txt) == "${text_1}${text_2}" ]] || die "append to t3.txt failed" + + # Append, mmap + echo -n $text_2 | x append-mmap m3.txt + [[ $(cat m3.txt) == "${text_1}${text_2}" ]] || die "append to t3.txt with mmap failed" + + # Append, linux specific. + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + # ">>" is a bash feature and exporting LD_PRELOAD in xetfs_on doesn't + # affect the current bash process + bash -c "echo -n $text_2 >> l3.txt" + [[ $(cat l3.txt) == "${text_1}${text_2}" ]] || die "append to l3.txt with >> failed" + fi +) + +# Write at specific location +assert_is_pointer_file t4.txt +assert_is_pointer_file m4.txt +assert_is_pointer_file l4.txt + +( + xetfs_on + echo -n $text_2 | x writeat 10 t4.txt + [[ $(cat t4.txt) == "${text_ins_at_10}" ]] || die "write at to t4.txt failed" + + # With MMap + echo -n $text_2 | x writeat-mmap 10 m4.txt + [[ $(cat m4.txt) == "${text_ins_at_10}" ]] || die "write at to t4.txt failed" + + # With existing tool dd, linux specific. + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + echo -n $text_2 | dd of=l4.txt bs=1 seek=10 conv=notrunc + [[ $(cat l4.txt) == "${text_ins_at_10}" ]] || die "write at to l4.txt failed" + fi +) + +popd + +# Write to all the files at once. +git xet clone --lazy $remote repo_4 +pushd repo_4 + +for n in 1 2 3 4 ; do + assert_is_pointer_file t$n.txt + assert_is_pointer_file m$n.txt + assert_is_pointer_file l$n.txt +done + +( + xetfs_on + + # Regular write to all the files + echo -n $text_2 | x write t?.txt + + for n in 1 2 3 4 ; do + [[ $(cat t$n.txt) == "${text_2}" ]] || die "Bulk write failed." + done + + # MMap write to all the files at once + echo -n $text_2 | x write-mmap m*.txt + + for n in 1 2 3 4 ; do + [[ $(cat m$n.txt) == "${text_2}" ]] || die "Bulk write failed." + done +) + +popd + +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + # Write to all the files at once. + git xet clone --lazy $remote repo_4b + pushd repo_4b + + for n in 1 2 3 4 ; do + assert_is_pointer_file t$n.txt + assert_is_pointer_file m$n.txt + assert_is_pointer_file l$n.txt + done + + ( + xetfs_on + + # Regular write to all the files + bash -c "cat t?.txt ; echo -n $text_2 | x write t?.txt" + + for n in 1 2 3 4 ; do + [[ $(cat t$n.txt) == "${text_2}" ]] || die "Bulk write failed." + done + + # MMap write to all the files at once + bash -c "cat t?.txt ; echo -n $text_2 | x write-mmap m*.txt" + + for n in 1 2 3 4 ; do + [[ $(cat m$n.txt) == "${text_2}" ]] || die "Bulk write failed." + done + ) + + popd +fi + + +# Test all the sizes +git xet clone --lazy $remote repo_5 +pushd repo_5 + +for n in 1 2 3 4 ; do + assert_is_pointer_file t$n.txt + assert_is_pointer_file m$n.txt + assert_is_pointer_file l$n.txt +done + +( + xetfs_on + + # Regular write to all the files + echo -n $text_2 | x write t?.txt + + for n in 1 2 3 4 ; do + [[ $(cat t$n.txt) == "${text_2}" ]] || die "Bulk write failed." + done + + # MMap write to all the files at once + echo -n $text_2 | x write-mmap m*.txt + + for n in 1 2 3 4 ; do + [[ $(cat m$n.txt) == "${text_2}" ]] || die "Bulk write failed." + done +) diff --git a/xetldfs/tests/integration_tests/test_bulk_write.sh b/xetldfs/tests/integration_tests/test_bulk_write.sh new file mode 100644 index 00000000..9d4678ea --- /dev/null +++ b/xetldfs/tests/integration_tests/test_bulk_write.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -e +set -x + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" &>/dev/null && pwd 2>/dev/null)" +. "$SCRIPT_DIR/initialize.sh" + +# This gives us $remote, $text_1, $text_1_len, $text_2, $text_2_len, $all_file_text, etc. +# It also sets XET_CAS_SIZETHRESHOLD such that $text_1 is stored in CAS and as a pointer file. +# +# Repo from in $remote has 12 files, t1.txt, t2.txt, t3.txt, t4.txt, and same for l and m instead of t. +# All are stored as pointer files and resolve in content to $text_1. +. "$SCRIPT_DIR/setup_test_repo.sh" + +# Write to all the files at once. +git xet clone --lazy $remote repo +pushd repo + +for n in 1 2 3 4 ; do + assert_is_pointer_file t$n.txt + assert_is_pointer_file m$n.txt + assert_is_pointer_file l$n.txt +done + +( + xetfs_on + + # Regular write to all the files + echo -n $text_2 | x write t?.txt + + for n in 1 2 3 4 ; do + [[ $(cat t$n.txt) == "${text_2}" ]] || die "Bulk write failed." + done + + # MMap write to all the files at once + echo -n $text_2 | x write-mmap m*.txt + + for n in 1 2 3 4 ; do + [[ $(cat m$n.txt) == "${text_2}" ]] || die "Bulk write failed." + done +) + +popd + +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + # Write to all the files at once. + git xet clone --lazy $remote repo_b + pushd repo_b + + for n in 1 2 3 4 ; do + assert_is_pointer_file t$n.txt + assert_is_pointer_file m$n.txt + assert_is_pointer_file l$n.txt + done + + ( + xetfs_on + + # Regular write to all the files + bash -c "cat t?.txt ; echo -n $text_2 | x write t?.txt" + + for n in 1 2 3 4 ; do + [[ $(cat t$n.txt) == "${text_2}" ]] || die "Bulk write failed." + done + + # MMap write to all the files at once + bash -c "cat t?.txt ; echo -n $text_2 | x write-mmap m*.txt" + + for n in 1 2 3 4 ; do + [[ $(cat m$n.txt) == "${text_2}" ]] || die "Bulk write failed." + done + ) + + popd +fi