|
36 | 36 | pkgA |
37 | 37 | pkgB |
38 | 38 | pkgC |
| 39 | + pkgs.coreutils |
39 | 40 | ]; |
40 | 41 | environment.systemPackages = [ pkgs.minio-client ]; |
| 42 | + nix.nixPath = [ "nixpkgs=${pkgs.path}" ]; |
41 | 43 | nix.extraOptions = '' |
42 | 44 | experimental-features = nix-command |
43 | 45 | substituters = |
|
667 | 669 |
|
668 | 670 | print(" ✓ Small file uploaded and verified") |
669 | 671 |
|
| 672 | + @setup_s3() |
| 673 | + def test_multipart_with_log_compression(bucket): |
| 674 | + """Test multipart upload with compressed build logs""" |
| 675 | + print("\n--- Test: Multipart Upload with Log Compression ---") |
| 676 | +
|
| 677 | + # Create a derivation that produces a large text log (12 MB of base64 output) |
| 678 | + drv_path = server.succeed( |
| 679 | + """ |
| 680 | + nix-instantiate --expr ' |
| 681 | + let pkgs = import <nixpkgs> {}; |
| 682 | + in derivation { |
| 683 | + name = "large-log-builder"; |
| 684 | + builder = "/bin/sh"; |
| 685 | + args = ["-c" "$coreutils/bin/dd if=/dev/urandom bs=1M count=12 | $coreutils/bin/base64; echo success > $out"]; |
| 686 | + coreutils = pkgs.coreutils; |
| 687 | + system = builtins.currentSystem; |
| 688 | + } |
| 689 | + ' |
| 690 | + """ |
| 691 | + ).strip() |
| 692 | +
|
| 693 | + print(" Building derivation to generate large log") |
| 694 | + server.succeed(f"nix-store --realize {drv_path} &>/dev/null") |
| 695 | +
|
| 696 | + # Upload logs with compression and multipart |
| 697 | + store_url = make_s3_url( |
| 698 | + bucket, |
| 699 | + **{ |
| 700 | + "multipart-upload": "true", |
| 701 | + "multipart-threshold": str(5 * 1024 * 1024), |
| 702 | + "multipart-chunk-size": str(5 * 1024 * 1024), |
| 703 | + "log-compression": "xz", |
| 704 | + } |
| 705 | + ) |
| 706 | +
|
| 707 | + print(" Uploading build log with compression and multipart") |
| 708 | + output = server.succeed( |
| 709 | + f"{ENV_WITH_CREDS} nix store copy-log --to '{store_url}' {drv_path} --debug 2>&1" |
| 710 | + ) |
| 711 | +
|
| 712 | + # Should use multipart for the compressed log |
| 713 | + if "using S3 multipart upload" not in output or "log/" not in output: |
| 714 | + print("Debug output:") |
| 715 | + print(output) |
| 716 | + raise Exception("Expected multipart upload to be used for compressed log") |
| 717 | +
|
| 718 | + if "parts uploaded" not in output: |
| 719 | + print("Debug output:") |
| 720 | + print(output) |
| 721 | + raise Exception("Expected multipart completion message") |
| 722 | +
|
| 723 | + print(" ✓ Compressed log uploaded with multipart") |
| 724 | +
|
670 | 725 | # ============================================================================ |
671 | 726 | # Main Test Execution |
672 | 727 | # ============================================================================ |
|
698 | 753 | test_nix_prefetch_url() |
699 | 754 | test_multipart_upload_basic() |
700 | 755 | test_multipart_threshold() |
| 756 | + test_multipart_with_log_compression() |
701 | 757 |
|
702 | 758 | print("\n" + "="*80) |
703 | 759 | print("✓ All S3 Binary Cache Store Tests Passed!") |
|
0 commit comments