From 45faf172175e563bebd161c24543f831549e0ec1 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Fri, 7 Nov 2025 16:53:00 +0100 Subject: [PATCH 01/18] poc for translating mirror urls --- src/main.zig | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/src/main.zig b/src/main.zig index 0884370..8b49114 100644 --- a/src/main.zig +++ b/src/main.zig @@ -436,13 +436,19 @@ pub fn main() !void { const url = try getVersionUrl(arena, app_data_path, semantic_version); defer url.deinit(arena); + + const filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; + const mirror_url = try MirrorUrls.getUrl(arena, app_data_path, filename); + // log.info("{f}", .{std.json.fmt(mirror_urls.list.items, .{ .whitespace = .indent_4 })}); + const hash = hashAndPath(try cmdFetch( gpa, arena, global_cache_directory, - url.fetch, + mirror_url, .{ .debug_hash = false }, )); + log.info("downloaded {s} to '{}{s}'", .{ hashstore_name, global_cache_directory, hash.path() }); if (maybe_hash) |*previous_hash| { if (previous_hash.val.eql(&hash.val)) { @@ -873,6 +879,54 @@ fn makeOfficialUrl(arena: Allocator, semantic_version: SemanticVersion) Download }; } +pub const MirrorUrls = struct { + list: std.ArrayListUnmanaged([]const u8) = .empty, + pub const mirrorlist = struct { + pub const url = "https://ziglang.org/download/community-mirrors.txt"; + pub const uri = std.Uri.parse(url) catch unreachable; + }; + + fn getUrl( + arena: Allocator, + app_data_path: []const u8, + filename: []const u8, + ) ![]const u8 { + const self = try get(arena, app_data_path); + assert(self.list.items.len > 0); + return std.fmt.allocPrint(arena, "{s}{s}{s}?source=anyzig", .{ + self.list.items[0], + if (std.mem.endsWith(u8, self.list.items[0], "/")) "" else "/", + filename, + }); + } + + fn get( + arena: Allocator, + app_data_path: []const u8, + ) !@This() { + const mirrors_path = try std.fs.path.join(arena, &.{ app_data_path, "community-mirrors.txt" }); + var self: @This() = .{}; + + try fetchFile(arena, mirrorlist.url, mirrorlist.uri, mirrors_path); + + const mirrors_content = blk: { + // since we just downloaded the file, this should always succeed now + const file = try std.fs.cwd().openFile(mirrors_path, .{}); + defer file.close(); + break :blk try file.readToEndAlloc(arena, std.math.maxInt(usize)); + }; + var iter = std.mem.splitScalar(u8, mirrors_content, '\n'); + while (iter.next()) |mirror| { + if (std.mem.startsWith(u8, mirror, "http")) { + try self.list.append(arena, mirror); + } + } + var rand = std.Random.DefaultPrng.init(@bitCast(std.time.timestamp())); + rand.random().shuffle([]const u8, self.list.items); + return self; + } +}; + fn getVersionUrl( arena: Allocator, app_data_path: []const u8, From ca9146580aba339876ec2615a85fe6b74ccebead Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Fri, 7 Nov 2025 17:32:26 +0100 Subject: [PATCH 02/18] add minizign dependency --- build.zig | 7 ++++++- build.zig.zon | 4 ++++ src/main.zig | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/build.zig b/build.zig index 78105f4..3514b72 100644 --- a/build.zig +++ b/build.zig @@ -6,6 +6,8 @@ const Exe = enum { zig, zls }; pub fn build(b: *std.Build) !void { const zig_dep = b.dependency("zig", .{}); + const minizign_dep = b.dependency("minizign", .{}); + const minizign_mod = minizign_dep.module("minizign"); const version_option: ?[11]u8 = if (b.option( []const u8, @@ -47,6 +49,7 @@ pub fn build(b: *std.Build) !void { .single_threaded = true, .imports = &.{ .{ .name = "zig", .module = zig_mod }, + .{ .name = "minizign", .module = minizign_mod }, .{ .name = "version", .module = dev_version_embed }, }, }), @@ -106,7 +109,7 @@ pub fn build(b: *std.Build) !void { ci_step.dependOn(test_step); ci_step.dependOn(&install_version_release_file.step); - try ci(b, &release_version, release_version_embed, zig_mod, ci_step, host_zip_exe); + try ci(b, &release_version, release_version_embed, zig_mod, minizign_mod, ci_step, host_zip_exe); } fn verifyForceVersion(v: []const u8) [11]u8 { @@ -593,6 +596,7 @@ fn ci( release_version: []const u8, release_version_embed: *std.Build.Module, zig_mod: *std.Build.Module, + minizign_mod: *std.Build.Module, ci_step: *std.Build.Step, host_zip_exe: *std.Build.Step.Compile, ) !void { @@ -633,6 +637,7 @@ fn ci( .single_threaded = true, .imports = &.{ .{ .name = "zig", .module = zig_mod }, + .{ .name = "minizign", .module = minizign_mod }, .{ .name = "version", .module = release_version_embed }, }, }), diff --git a/build.zig.zon b/build.zig.zon index b26ede4..867c343 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -12,6 +12,10 @@ .url = "git+https://github.com/marler8997/zipcmdline#3dfca786a489d117e4b72ea10ffb4bbd9fc2dd72", .hash = "12201a08d7eff7619c8eb8284691a3ff959861b4bdd87216f180ed136672fb4ea26f", }, + .minizign = .{ + .url = "git+https://github.com/jedisct1/zig-minisign.git#50af1aa2ca0a4635e742b0a83735bccc3052b932", + .hash = "minizign-0.1.7-gg18cWlFAgBHo9NOoJiCzEn3dgmdE2bpWfQOuVipDhsk", + }, }, .paths = .{ "build.zig", diff --git a/src/main.zig b/src/main.zig index 8b49114..c6f911a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -17,6 +17,7 @@ const Directory = std.Build.Cache.Directory; const EnvVar = std.zig.EnvVar; const zig = @import("zig"); +const minizign = @import("minizign"); const Package = zig.Package; const introspect = zig.introspect; From 508948ecc0c7aeba463e8c156ca8f11899d1c799 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Fri, 7 Nov 2025 18:38:23 +0100 Subject: [PATCH 03/18] wip on minisign validation --- build.zig.zon | 6 +-- src/main.zig | 105 ++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 80 insertions(+), 31 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 867c343..efedc83 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -2,11 +2,11 @@ .name = .anyzig, .version = "0.0.0", .fingerprint = 0x7cd092f5bd0fed33, // Changing this has security and trust implications. - .minimum_zig_version = "0.14.0", + .minimum_zig_version = "0.14.1", .dependencies = .{ .zig = .{ - .url = "git+https://github.com/ziglang/zig#5ad91a646a753cc3eecd8751e61cf458dadd9ac4", - .hash = "zig-0.0.0-Fp4XJDTgLgvQHn6QVvTe6INRdgdAzu43qE4JBXxhn9Ln", + .url = "git+https://github.com/ziglang/zig.git?ref=0.14.1#d03a147ea0a590ca711b3db07106effc559b0fc6", + .hash = "zig-0.0.0-Fp4XJConMAvbptegZi4D-tp5r9VKMY6KCB_mEm9mSzZN", }, .zip = .{ .url = "git+https://github.com/marler8997/zipcmdline#3dfca786a489d117e4b72ea10ffb4bbd9fc2dd72", diff --git a/src/main.zig b/src/main.zig index c6f911a..44a0542 100644 --- a/src/main.zig +++ b/src/main.zig @@ -434,19 +434,29 @@ pub fn main() !void { else => |e| return e, } } + // TODO: cache mirror list + var mirrors = try MirrorUrls.get(gpa, app_data_path); + defer mirrors.deinit(gpa); + assert(mirrors.list.items.len > 0); const url = try getVersionUrl(arena, app_data_path, semantic_version); defer url.deinit(arena); - const filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; - const mirror_url = try MirrorUrls.getUrl(arena, app_data_path, filename); - // log.info("{f}", .{std.json.fmt(mirror_urls.list.items, .{ .whitespace = .indent_4 })}); + // TODO: retries + const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; + const mirror = try FetchInfo.init(gpa, app_data_path, mirrors.list.items[0], zig_archive_filename); + defer mirror.deinit(gpa); + + defer std.fs.cwd().deleteFile(mirror.archive_path) catch {}; + try fetchFile(arena, mirror.archive_url, try std.Uri.parse(mirror.archive_url), mirror.archive_path); + + // TODO: download and verify minizign signature const hash = hashAndPath(try cmdFetch( gpa, arena, global_cache_directory, - mirror_url, + mirror.archive_path, .{ .debug_hash = false }, )); @@ -880,52 +890,91 @@ fn makeOfficialUrl(arena: Allocator, semantic_version: SemanticVersion) Download }; } -pub const MirrorUrls = struct { - list: std.ArrayListUnmanaged([]const u8) = .empty, - pub const mirrorlist = struct { - pub const url = "https://ziglang.org/download/community-mirrors.txt"; - pub const uri = std.Uri.parse(url) catch unreachable; - }; - - fn getUrl( - arena: Allocator, - app_data_path: []const u8, - filename: []const u8, - ) ![]const u8 { - const self = try get(arena, app_data_path); - assert(self.list.items.len > 0); - return std.fmt.allocPrint(arena, "{s}{s}{s}?source=anyzig", .{ - self.list.items[0], - if (std.mem.endsWith(u8, self.list.items[0], "/")) "" else "/", +const FetchInfo = struct { + archive_url: []const u8, + archive_path: []const u8, + minisign_url: []const u8, + minisign_path: []const u8, + fn init(gpa: Allocator, tmpdir: []const u8, mirror_url: []const u8, filename: []const u8) !@This() { + const archive_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}?source=anyzig", .{ + mirror_url, + if (std.mem.endsWith(u8, mirror_url, "/")) "" else "/", + filename, + }); + errdefer gpa.free(archive_url); + const minisign_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}.minisig?source=anyzig", .{ + mirror_url, + if (std.mem.endsWith(u8, mirror_url, "/")) "" else "/", filename, }); + errdefer gpa.free(minisign_url); + const minisign_filename = try std.mem.concat(gpa, u8, &.{ filename, ".minisig" }); + defer gpa.free(minisign_filename); + + const minisign_path = try std.fs.path.join(gpa, &.{ + tmpdir, + minisign_filename, + }); + return .{ + .archive_url = archive_url, + .minisign_url = minisign_url, + .minisign_path = minisign_path, + .archive_path = minisign_path[0 .. minisign_path.len - ".minisig".len], + }; + } + + fn deinit(self: @This(), gpa: Allocator) void { + gpa.free(self.archive_url); + gpa.free(self.minisign_url); + gpa.free(self.minisign_path); + // do not free archive_path here, since its a slice of minisign_path } +}; + +pub const MirrorUrls = struct { + list: std.ArrayListUnmanaged([]const u8) = .empty, + const mirrorlist = struct { + const url = "https://ziglang.org/download/community-mirrors.txt"; + const uri = std.Uri.parse(url) catch unreachable; + }; fn get( - arena: Allocator, - app_data_path: []const u8, + gpa: Allocator, + tmpdir: []const u8, ) !@This() { - const mirrors_path = try std.fs.path.join(arena, &.{ app_data_path, "community-mirrors.txt" }); + const mirrors_path = try std.fs.path.join(gpa, &.{ tmpdir, "community-mirrors.txt" }); + defer gpa.free(mirrors_path); var self: @This() = .{}; - try fetchFile(arena, mirrorlist.url, mirrorlist.uri, mirrors_path); + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + try fetchFile(arena.allocator(), mirrorlist.url, mirrorlist.uri, mirrors_path); const mirrors_content = blk: { // since we just downloaded the file, this should always succeed now const file = try std.fs.cwd().openFile(mirrors_path, .{}); defer file.close(); - break :blk try file.readToEndAlloc(arena, std.math.maxInt(usize)); + break :blk try file.readToEndAlloc(gpa, std.math.maxInt(usize)); }; + defer gpa.free(mirrors_content); + var iter = std.mem.splitScalar(u8, mirrors_content, '\n'); while (iter.next()) |mirror| { if (std.mem.startsWith(u8, mirror, "http")) { - try self.list.append(arena, mirror); + try self.list.append(gpa, try gpa.dupe(u8, mirror)); } } var rand = std.Random.DefaultPrng.init(@bitCast(std.time.timestamp())); rand.random().shuffle([]const u8, self.list.items); return self; } + + fn deinit(self: *@This(), gpa: Allocator) void { + for (self.list.items) |mirror| { + gpa.free(mirror); + } + self.list.deinit(gpa); + } }; fn getVersionUrl( @@ -1065,7 +1114,7 @@ fn fetchFile( const progress_node_name = std.fmt.allocPrint(scratch, "fetch {s}", .{uri}) catch |e| oom(e); defer scratch.free(progress_node_name); - const node = root.start(progress_node_name, 1); + const node = root.start(progress_node_name, 0); defer node.end(); const lock_filepath = try std.mem.concat(scratch, u8, &.{ out_filepath, ".lock" }); From fec857e0c8d45814144a3663d0c8e4f435c3a6f9 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Fri, 7 Nov 2025 19:32:53 +0100 Subject: [PATCH 04/18] add minisign validation --- src/main.zig | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/main.zig b/src/main.zig index 44a0542..d7831b3 100644 --- a/src/main.zig +++ b/src/main.zig @@ -444,19 +444,21 @@ pub fn main() !void { // TODO: retries const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; - const mirror = try FetchInfo.init(gpa, app_data_path, mirrors.list.items[0], zig_archive_filename); - defer mirror.deinit(gpa); + const fetchinfo = try FetchInfo.init(gpa, app_data_path, mirrors.list.items[0], zig_archive_filename); + defer fetchinfo.deinit(gpa); - defer std.fs.cwd().deleteFile(mirror.archive_path) catch {}; - try fetchFile(arena, mirror.archive_url, try std.Uri.parse(mirror.archive_url), mirror.archive_path); + defer std.fs.cwd().deleteFile(fetchinfo.archive_path) catch {}; + try fetchFile(arena, fetchinfo.archive_url, try std.Uri.parse(fetchinfo.archive_url), fetchinfo.archive_path); + defer std.fs.cwd().deleteFile(fetchinfo.minisign_path) catch {}; + try fetchFile(arena, fetchinfo.minisign_url, try std.Uri.parse(fetchinfo.minisign_url), fetchinfo.minisign_path); - // TODO: download and verify minizign signature + try fetchinfo.validateMinisign(gpa); const hash = hashAndPath(try cmdFetch( gpa, arena, global_cache_directory, - mirror.archive_path, + fetchinfo.archive_path, .{ .debug_hash = false }, )); @@ -923,6 +925,25 @@ const FetchInfo = struct { }; } + const zig_org_minisign_pubkey = minizign.PublicKey.decodeFromBase64("RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U") catch unreachable; + fn validateMinisign(self: @This(), gpa: Allocator) !void { + const sig_bytes = try std.fs.cwd().readFileAlloc(gpa, self.minisign_path, std.math.maxInt(u32)); + defer gpa.free(sig_bytes); + + const archive_file = try std.fs.cwd().openFile(self.archive_path, .{}); + defer archive_file.close(); + + var sig = try minizign.Signature.decode(gpa, sig_bytes); + defer sig.deinit(); + + try zig_org_minisign_pubkey.verifyFile( + gpa, + archive_file, + sig, + null, + ); + } + fn deinit(self: @This(), gpa: Allocator) void { gpa.free(self.archive_url); gpa.free(self.minisign_url); From 7f70467d0951c5167439d59a49642cd777477a09 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Fri, 7 Nov 2025 20:16:05 +0100 Subject: [PATCH 05/18] fix zls builds --- build.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/build.zig b/build.zig index 3514b72..5c16f90 100644 --- a/build.zig +++ b/build.zig @@ -655,6 +655,7 @@ fn ci( .single_threaded = true, .imports = &.{ .{ .name = "zig", .module = zig_mod }, + .{ .name = "minizign", .module = minizign_mod }, .{ .name = "version", .module = release_version_embed }, }, }), From f312f008653987f3a994ce1ad791dae1f6661f12 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Fri, 7 Nov 2025 21:05:48 +0100 Subject: [PATCH 06/18] implement mirror retries --- src/main.zig | 52 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/src/main.zig b/src/main.zig index d7831b3..5a98430 100644 --- a/src/main.zig +++ b/src/main.zig @@ -442,18 +442,22 @@ pub fn main() !void { const url = try getVersionUrl(arena, app_data_path, semantic_version); defer url.deinit(arena); - // TODO: retries const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; - const fetchinfo = try FetchInfo.init(gpa, app_data_path, mirrors.list.items[0], zig_archive_filename); + var fetchinfo: FetchInfo = undefined; + var fetch_successful: bool = false; + for (mirrors.list.items) |mirror_url| { + fetchinfo = try FetchInfo.init(gpa, app_data_path, mirror_url, zig_archive_filename); + errdefer fetchinfo.deinit(gpa); + fetchinfo.fetchAndValidate(gpa) catch continue; + fetch_successful = true; + break; + } + if (!fetch_successful) { + std.log.err("ran out of mirrors for: {s}", .{zig_archive_filename}); + return error.NoMoreMirrors; + } defer fetchinfo.deinit(gpa); - defer std.fs.cwd().deleteFile(fetchinfo.archive_path) catch {}; - try fetchFile(arena, fetchinfo.archive_url, try std.Uri.parse(fetchinfo.archive_url), fetchinfo.archive_path); - defer std.fs.cwd().deleteFile(fetchinfo.minisign_path) catch {}; - try fetchFile(arena, fetchinfo.minisign_url, try std.Uri.parse(fetchinfo.minisign_url), fetchinfo.minisign_path); - - try fetchinfo.validateMinisign(gpa); - const hash = hashAndPath(try cmdFetch( gpa, arena, @@ -910,7 +914,9 @@ const FetchInfo = struct { filename, }); errdefer gpa.free(minisign_url); - const minisign_filename = try std.mem.concat(gpa, u8, &.{ filename, ".minisig" }); + var rand = std.Random.DefaultPrng.init(@bitCast(std.time.timestamp())); + + const minisign_filename = try std.fmt.allocPrint(gpa, "tmp-{x}-{s}{s}", .{ rand.random().int(u32), filename, ".minisig" }); defer gpa.free(minisign_filename); const minisign_path = try std.fs.path.join(gpa, &.{ @@ -925,6 +931,25 @@ const FetchInfo = struct { }; } + fn deinit(self: @This(), gpa: Allocator) void { + std.fs.cwd().deleteFile(self.archive_path) catch {}; + std.fs.cwd().deleteFile(self.minisign_path) catch {}; + gpa.free(self.archive_url); + gpa.free(self.minisign_url); + gpa.free(self.minisign_path); + // do not free archive_path here, since its a slice of minisign_path + } + + fn fetchAndValidate(self: @This(), gpa: Allocator) !void { + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + + try fetchFile(arena.allocator(), self.archive_url, try std.Uri.parse(self.archive_url), self.archive_path); + try fetchFile(arena.allocator(), self.minisign_url, try std.Uri.parse(self.minisign_url), self.minisign_path); + + try self.validateMinisign(gpa); + } + const zig_org_minisign_pubkey = minizign.PublicKey.decodeFromBase64("RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U") catch unreachable; fn validateMinisign(self: @This(), gpa: Allocator) !void { const sig_bytes = try std.fs.cwd().readFileAlloc(gpa, self.minisign_path, std.math.maxInt(u32)); @@ -943,13 +968,6 @@ const FetchInfo = struct { null, ); } - - fn deinit(self: @This(), gpa: Allocator) void { - gpa.free(self.archive_url); - gpa.free(self.minisign_url); - gpa.free(self.minisign_path); - // do not free archive_path here, since its a slice of minisign_path - } }; pub const MirrorUrls = struct { From 3ecbb8e0210758fc5aa0238725d50c7180067f59 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Fri, 7 Nov 2025 21:24:00 +0100 Subject: [PATCH 07/18] better error messages - fetchFile still needs error handling --- src/main.zig | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main.zig b/src/main.zig index 5a98430..1cc0263 100644 --- a/src/main.zig +++ b/src/main.zig @@ -944,10 +944,29 @@ const FetchInfo = struct { var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); - try fetchFile(arena.allocator(), self.archive_url, try std.Uri.parse(self.archive_url), self.archive_path); - try fetchFile(arena.allocator(), self.minisign_url, try std.Uri.parse(self.minisign_url), self.minisign_path); + fetchFile( + arena.allocator(), + self.archive_url, + try std.Uri.parse(self.archive_url), + self.archive_path, + ) catch |err| { + std.log.err("failed to download archive: {} {s}", .{ err, self.archive_url }); + return err; + }; + fetchFile( + arena.allocator(), + self.minisign_url, + try std.Uri.parse(self.minisign_url), + self.minisign_path, + ) catch |err| { + std.log.err("failed to download signature: {} {s}", .{ err, self.minisign_url }); + return err; + }; - try self.validateMinisign(gpa); + self.validateMinisign(gpa) catch |err| { + std.log.err("failed to validate: {} {s}", .{ err, self.archive_url }); + return err; + }; } const zig_org_minisign_pubkey = minizign.PublicKey.decodeFromBase64("RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U") catch unreachable; @@ -1321,7 +1340,7 @@ pub fn cmdFetch( }; defer fetch.deinit(); - log.info("downloading '{s}'...", .{url}); + log.info("cacheing '{s}'...", .{url}); fetch.run() catch |err| switch (err) { error.OutOfMemory => errExit("out of memory", .{}), error.FetchFailed => {}, // error bundle checked below From cf25a477630b719d627e4657ac50cf20370ad172 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Fri, 7 Nov 2025 23:16:25 +0100 Subject: [PATCH 08/18] return more errors from fetchFile --- src/main.zig | 64 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/src/main.zig b/src/main.zig index 1cc0263..0c93a6f 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1184,39 +1184,53 @@ fn fetchFile( var client = std.http.Client{ .allocator = scratch }; defer client.deinit(); - client.initDefaultProxies(scratch) catch |err| std.debug.panic( - "fetch '{}': init proxy failed with {s}", - .{ uri, @errorName(err) }, - ); + client.initDefaultProxies(scratch) catch |err| { + log.err( + "fetch '{}': init proxy failed with {s}", + .{ uri, @errorName(err) }, + ); + return err; + }; + var header_buffer: [4096]u8 = undefined; - var request = client.open(.GET, uri, .{ + var request = try client.open(.GET, uri, .{ .server_header_buffer = &header_buffer, .keep_alive = false, - }) catch |e| std.debug.panic( - "fetch '{}': connect failed with {s}", - .{ uri, @errorName(e) }, - ); + }); defer request.deinit(); - request.send() catch |e| std.debug.panic( - "fetch '{}': send failed with {s}", - .{ uri, @errorName(e) }, - ); - request.wait() catch |e| std.debug.panic( - "fetch '{}': wait failed with {s}", - .{ uri, @errorName(e) }, - ); - if (request.response.status != .ok) return errExit( - "fetch '{}': HTTP response {} \"{?s}\"", - .{ uri, @intFromEnum(request.response.status), request.response.status.phrase() }, - ); + request.send() catch |e| { + log.err( + "fetch '{}': send failed with {s}", + .{ uri, @errorName(e) }, + ); + return e; + }; + request.wait() catch |e| { + std.debug.panic( + "fetch '{}': wait failed with {s}", + .{ uri, @errorName(e) }, + ); + return e; + }; + + if (request.response.status != .ok) { + log.err( + "fetch '{}': HTTP response {} \"{?s}\"", + .{ uri, @intFromEnum(request.response.status), request.response.status.phrase() }, + ); + return error.BadHttpStatus; + } const out_filepath_tmp = std.mem.concat(scratch, u8, &.{ out_filepath, ".fetching" }) catch |e| oom(e); defer scratch.free(out_filepath_tmp); - const file = std.fs.cwd().createFile(out_filepath_tmp, .{}) catch |e| std.debug.panic( - "create '{s}' failed with {s}", - .{ out_filepath_tmp, @errorName(e) }, - ); + const file = std.fs.cwd().createFile(out_filepath_tmp, .{}) catch |e| { + std.log.err( + "create '{s}' failed with {s}", + .{ out_filepath_tmp, @errorName(e) }, + ); + return error.CouldNotCreateFile; + }; defer { if (std.fs.cwd().deleteFile(out_filepath_tmp)) { std.log.info("removed '{s}'", .{out_filepath_tmp}); From f9fbd72435483be8bbbc8df8cf92c6630c08d267 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Sat, 8 Nov 2025 07:39:48 +0100 Subject: [PATCH 09/18] fix memory leak in FetchInfo error cases --- src/main.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index 0c93a6f..cabdaa1 100644 --- a/src/main.zig +++ b/src/main.zig @@ -447,8 +447,10 @@ pub fn main() !void { var fetch_successful: bool = false; for (mirrors.list.items) |mirror_url| { fetchinfo = try FetchInfo.init(gpa, app_data_path, mirror_url, zig_archive_filename); - errdefer fetchinfo.deinit(gpa); - fetchinfo.fetchAndValidate(gpa) catch continue; + fetchinfo.fetchAndValidate(gpa) catch { + fetchinfo.deinit(gpa); + continue; + }; fetch_successful = true; break; } From 49299d15193651d1b2cca439068ee05517b00fcf Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Sat, 8 Nov 2025 10:43:54 +0100 Subject: [PATCH 10/18] move mirror loop out of main --- src/main.zig | 53 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/main.zig b/src/main.zig index cabdaa1..84b863d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -443,21 +443,8 @@ pub fn main() !void { defer url.deinit(arena); const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; - var fetchinfo: FetchInfo = undefined; - var fetch_successful: bool = false; - for (mirrors.list.items) |mirror_url| { - fetchinfo = try FetchInfo.init(gpa, app_data_path, mirror_url, zig_archive_filename); - fetchinfo.fetchAndValidate(gpa) catch { - fetchinfo.deinit(gpa); - continue; - }; - fetch_successful = true; - break; - } - if (!fetch_successful) { - std.log.err("ran out of mirrors for: {s}", .{zig_archive_filename}); - return error.NoMoreMirrors; - } + + const fetchinfo = try mirrors.fetchFromAny(gpa, app_data_path, zig_archive_filename); defer fetchinfo.deinit(gpa); const hash = hashAndPath(try cmdFetch( @@ -903,27 +890,29 @@ const FetchInfo = struct { archive_path: []const u8, minisign_url: []const u8, minisign_path: []const u8, + const anyzig_mirror_url_query = "source=anyzig/" ++ @embedFile("version"); fn init(gpa: Allocator, tmpdir: []const u8, mirror_url: []const u8, filename: []const u8) !@This() { - const archive_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}?source=anyzig", .{ + const archive_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}?{s}", .{ mirror_url, if (std.mem.endsWith(u8, mirror_url, "/")) "" else "/", filename, + anyzig_mirror_url_query, }); errdefer gpa.free(archive_url); - const minisign_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}.minisig?source=anyzig", .{ + const minisign_url = try std.fmt.allocPrint(gpa, "{s}{s}{s}.minisig?{s}", .{ mirror_url, if (std.mem.endsWith(u8, mirror_url, "/")) "" else "/", filename, + anyzig_mirror_url_query, }); errdefer gpa.free(minisign_url); - var rand = std.Random.DefaultPrng.init(@bitCast(std.time.timestamp())); - const minisign_filename = try std.fmt.allocPrint(gpa, "tmp-{x}-{s}{s}", .{ rand.random().int(u32), filename, ".minisig" }); - defer gpa.free(minisign_filename); + const minisig_filename = try std.fmt.allocPrint(gpa, "tmp-{s}{s}", .{ filename, ".minisig" }); + defer gpa.free(minisig_filename); const minisign_path = try std.fs.path.join(gpa, &.{ tmpdir, - minisign_filename, + minisig_filename, }); return .{ .archive_url = archive_url, @@ -1020,7 +1009,8 @@ pub const MirrorUrls = struct { var iter = std.mem.splitScalar(u8, mirrors_content, '\n'); while (iter.next()) |mirror| { - if (std.mem.startsWith(u8, mirror, "http")) { + const mirror_without_whitespace = std.mem.trim(u8, mirror, &std.ascii.whitespace); + if (mirror_without_whitespace.len > 0) { try self.list.append(gpa, try gpa.dupe(u8, mirror)); } } @@ -1029,6 +1019,25 @@ pub const MirrorUrls = struct { return self; } + fn fetchFromAny(self: @This(), gpa: Allocator, tmpdir: []const u8, filename: []const u8) !FetchInfo { + var fetchinfo: FetchInfo = undefined; + var fetch_successful: bool = false; + for (self.list.items) |mirror_url| { + fetchinfo = try FetchInfo.init(gpa, tmpdir, mirror_url, filename); + fetchinfo.fetchAndValidate(gpa) catch { + fetchinfo.deinit(gpa); + continue; + }; + fetch_successful = true; + break; + } + if (!fetch_successful) { + log.err("ran out of mirrors for: {s}", .{filename}); + return error.NoMoreMirrors; + } + return fetchinfo; + } + fn deinit(self: *@This(), gpa: Allocator) void { for (self.list.items) |mirror| { gpa.free(mirror); From c390c22c3994a8c593e88efc8bc89938a883c7a6 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Sun, 9 Nov 2025 13:49:01 +0100 Subject: [PATCH 11/18] add log messages when deleting tempfile fails --- src/main.zig | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index 84b863d..e4745cf 100644 --- a/src/main.zig +++ b/src/main.zig @@ -923,8 +923,14 @@ const FetchInfo = struct { } fn deinit(self: @This(), gpa: Allocator) void { - std.fs.cwd().deleteFile(self.archive_path) catch {}; - std.fs.cwd().deleteFile(self.minisign_path) catch {}; + std.fs.cwd().deleteFile(self.archive_path) catch |err| switch (err) { + error.FileNotFound => {}, + else => std.log.err("remove '{s}' failed with {s}", .{ self.archive_path, @errorName(err) }), + }; + std.fs.cwd().deleteFile(self.minisign_path) catch |err| switch (err) { + error.FileNotFound => {}, + else => std.log.err("remove '{s}' failed with {s}", .{ self.archive_path, @errorName(err) }), + }; gpa.free(self.archive_url); gpa.free(self.minisign_url); gpa.free(self.minisign_path); From 4a3b4ccf806c5c80e987b76eec0647326e70795f Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Sun, 9 Nov 2025 14:16:51 +0100 Subject: [PATCH 12/18] remove panics from fetchFile --- src/main.zig | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/main.zig b/src/main.zig index e4745cf..456e712 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1026,22 +1026,16 @@ pub const MirrorUrls = struct { } fn fetchFromAny(self: @This(), gpa: Allocator, tmpdir: []const u8, filename: []const u8) !FetchInfo { - var fetchinfo: FetchInfo = undefined; - var fetch_successful: bool = false; for (self.list.items) |mirror_url| { - fetchinfo = try FetchInfo.init(gpa, tmpdir, mirror_url, filename); + const fetchinfo = try FetchInfo.init(gpa, tmpdir, mirror_url, filename); fetchinfo.fetchAndValidate(gpa) catch { fetchinfo.deinit(gpa); continue; }; - fetch_successful = true; - break; - } - if (!fetch_successful) { - log.err("ran out of mirrors for: {s}", .{filename}); - return error.NoMoreMirrors; + return fetchinfo; } - return fetchinfo; + log.err("ran out of mirrors for: {s}", .{filename}); + return error.NoMoreMirrors; } fn deinit(self: *@This(), gpa: Allocator) void { @@ -1223,7 +1217,7 @@ fn fetchFile( return e; }; request.wait() catch |e| { - std.debug.panic( + log.err( "fetch '{}': wait failed with {s}", .{ uri, @errorName(e) }, ); @@ -1242,7 +1236,7 @@ fn fetchFile( defer scratch.free(out_filepath_tmp); const file = std.fs.cwd().createFile(out_filepath_tmp, .{}) catch |e| { - std.log.err( + log.err( "create '{s}' failed with {s}", .{ out_filepath_tmp, @errorName(e) }, ); @@ -1250,10 +1244,10 @@ fn fetchFile( }; defer { if (std.fs.cwd().deleteFile(out_filepath_tmp)) { - std.log.info("removed '{s}'", .{out_filepath_tmp}); + log.info("removed '{s}'", .{out_filepath_tmp}); } else |err| switch (err) { error.FileNotFound => {}, - else => |e| std.log.err("remove '{s}' failed with {s}", .{ out_filepath_tmp, @errorName(e) }), + else => |e| log.err("remove '{s}' failed with {s}", .{ out_filepath_tmp, @errorName(e) }), } file.close(); } @@ -1263,7 +1257,7 @@ fn fetchFile( // not sure if it's a problem with the mach server or Zig's HTTP client if (request.response.content_length) |content_length| { if (std.mem.eql(u8, url_string, DownloadIndexKind.mach.url())) { - std.log.warn("ignoring content length {} for mach index", .{content_length}); + log.warn("ignoring content length {} for mach index", .{content_length}); break :blk null; } } @@ -1285,10 +1279,13 @@ fn fetchFile( total_received += len; if (maybe_content_length) |content_length| { - if (total_received > content_length) errExit( - "fetch '{}': read more than Content-Length ({})", - .{ uri, content_length }, - ); + if (total_received > content_length) { + log.err( + "fetch '{}': read more than Content-Length ({})", + .{ uri, content_length }, + ); + return error.ContentLengthMismatch; + } } // NOTE: not going through a buffered writer since we're writing // large chunks From 04816ac1d2070d60a968affa058505e89791cbf7 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Sun, 9 Nov 2025 14:19:54 +0100 Subject: [PATCH 13/18] remove more panics from fetchFile --- src/main.zig | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/main.zig b/src/main.zig index 456e712..81763f4 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1271,10 +1271,13 @@ fn fetchFile( var total_received: u64 = 0; while (true) { var buf: [@max(std.heap.page_size_min, 4096)]u8 = undefined; - const len = request.reader().read(&buf) catch |e| std.debug.panic( - "fetch '{}': read failed with {s}", - .{ uri, @errorName(e) }, - ); + const len = request.reader().read(&buf) catch |e| { + log.err( + "fetch '{}': read failed with {s}", + .{ uri, @errorName(e) }, + ); + return e; + }; if (len == 0) break; total_received += len; @@ -1289,17 +1292,23 @@ fn fetchFile( } // NOTE: not going through a buffered writer since we're writing // large chunks - file.writer().writeAll(buf[0..len]) catch |err| std.debug.panic( - "fetch '{}': write {} bytes of HTTP response failed with {s}", - .{ uri, len, @errorName(err) }, - ); + file.writer().writeAll(buf[0..len]) catch |err| { + log.err( + "fetch '{}': write {} bytes of HTTP response failed with {s}", + .{ uri, len, @errorName(err) }, + ); + return err; + }; } if (maybe_content_length) |content_length| { - if (total_received != content_length) errExit( - "fetch '{}': Content-Length is {} but only read {}", - .{ uri, content_length, total_received }, - ); + if (total_received != content_length) { + log.err( + "fetch '{}': Content-Length is {} but only read {}", + .{ uri, content_length, total_received }, + ); + return error.ContentLengthMismatch; + } } try std.fs.cwd().rename(out_filepath_tmp, out_filepath); From bbb8af970130c1368f4bf94e5440de04c35c8e8b Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Sun, 9 Nov 2025 14:39:30 +0100 Subject: [PATCH 14/18] catch errors on mirrorlist download, so we can reuse an existing one --- src/main.zig | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main.zig b/src/main.zig index 81763f4..4aaed1d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -434,7 +434,6 @@ pub fn main() !void { else => |e| return e, } } - // TODO: cache mirror list var mirrors = try MirrorUrls.get(gpa, app_data_path); defer mirrors.deinit(gpa); assert(mirrors.list.items.len > 0); @@ -997,17 +996,19 @@ pub const MirrorUrls = struct { gpa: Allocator, tmpdir: []const u8, ) !@This() { - const mirrors_path = try std.fs.path.join(gpa, &.{ tmpdir, "community-mirrors.txt" }); - defer gpa.free(mirrors_path); + const mirrorlist_path = try std.fs.path.join(gpa, &.{ tmpdir, "community-mirrors.txt" }); + defer gpa.free(mirrorlist_path); var self: @This() = .{}; var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); - try fetchFile(arena.allocator(), mirrorlist.url, mirrorlist.uri, mirrors_path); + fetchFile(arena.allocator(), mirrorlist.url, mirrorlist.uri, mirrorlist_path) catch { + log.err("failed to fetch mirrorlist to: {s}", .{mirrorlist_path}); + }; const mirrors_content = blk: { - // since we just downloaded the file, this should always succeed now - const file = try std.fs.cwd().openFile(mirrors_path, .{}); + // load the mirrorlist we just downloaded, or is still around from a previous run + const file = try std.fs.cwd().openFile(mirrorlist_path, .{}); defer file.close(); break :blk try file.readToEndAlloc(gpa, std.math.maxInt(usize)); }; From 1a3a9a6977e57915ce533b13d5e9c4721798355a Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Sun, 9 Nov 2025 15:13:10 +0100 Subject: [PATCH 15/18] put downloads in a subdirectory --- src/main.zig | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main.zig b/src/main.zig index 4aaed1d..9e5ce2f 100644 --- a/src/main.zig +++ b/src/main.zig @@ -443,7 +443,8 @@ pub fn main() !void { const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; - const fetchinfo = try mirrors.fetchFromAny(gpa, app_data_path, zig_archive_filename); + const download_path = try std.fs.path.join(arena, &.{ app_data_path, "download" }); + const fetchinfo = try mirrors.fetchFromAny(gpa, download_path, zig_archive_filename); defer fetchinfo.deinit(gpa); const hash = hashAndPath(try cmdFetch( @@ -906,7 +907,7 @@ const FetchInfo = struct { }); errdefer gpa.free(minisign_url); - const minisig_filename = try std.fmt.allocPrint(gpa, "tmp-{s}{s}", .{ filename, ".minisig" }); + const minisig_filename = try std.fmt.allocPrint(gpa, "{s}{s}", .{ filename, ".minisig" }); defer gpa.free(minisig_filename); const minisign_path = try std.fs.path.join(gpa, &.{ @@ -928,7 +929,7 @@ const FetchInfo = struct { }; std.fs.cwd().deleteFile(self.minisign_path) catch |err| switch (err) { error.FileNotFound => {}, - else => std.log.err("remove '{s}' failed with {s}", .{ self.archive_path, @errorName(err) }), + else => std.log.err("remove '{s}' failed with {s}", .{ self.minisign_path, @errorName(err) }), }; gpa.free(self.archive_url); gpa.free(self.minisign_url); From 29b4bc23ae4d3926321c1a04fe34278a0b9c9e54 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 10 Nov 2025 09:31:32 +0100 Subject: [PATCH 16/18] only use mirrors on downloads from ziglang.org --- src/main.zig | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main.zig b/src/main.zig index 9e5ce2f..d7ccc2a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -434,24 +434,31 @@ pub fn main() !void { else => |e| return e, } } - var mirrors = try MirrorUrls.get(gpa, app_data_path); - defer mirrors.deinit(gpa); - assert(mirrors.list.items.len > 0); - const url = try getVersionUrl(arena, app_data_path, semantic_version); + var url = try getVersionUrl(arena, app_data_path, semantic_version); defer url.deinit(arena); - const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; + const fetchinfo: ?FetchInfo = if (std.mem.startsWith(u8, url.fetch, "https://ziglang.org")) fetchinfo: { + var mirrors = try MirrorUrls.get(gpa, app_data_path); + defer mirrors.deinit(gpa); + assert(mirrors.list.items.len > 0); - const download_path = try std.fs.path.join(arena, &.{ app_data_path, "download" }); - const fetchinfo = try mirrors.fetchFromAny(gpa, download_path, zig_archive_filename); - defer fetchinfo.deinit(gpa); + const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; + + const download_path = try std.fs.path.join(arena, &.{ app_data_path, "download" }); + const fi = try mirrors.fetchFromAny(gpa, download_path, zig_archive_filename); + url.fetch = fi.archive_path; + break :fetchinfo fi; + } else null; + defer { + if (fetchinfo) |fi| fi.deinit(gpa); + } const hash = hashAndPath(try cmdFetch( gpa, arena, global_cache_directory, - fetchinfo.archive_path, + url.fetch, .{ .debug_hash = false }, )); From c46e3a707bf7bd78998089511281481fd660d2f8 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 10 Nov 2025 09:37:17 +0100 Subject: [PATCH 17/18] handle error on fetching mirrorlist --- src/main.zig | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main.zig b/src/main.zig index d7ccc2a..7bc4533 100644 --- a/src/main.zig +++ b/src/main.zig @@ -438,10 +438,15 @@ pub fn main() !void { var url = try getVersionUrl(arena, app_data_path, semantic_version); defer url.deinit(arena); - const fetchinfo: ?FetchInfo = if (std.mem.startsWith(u8, url.fetch, "https://ziglang.org")) fetchinfo: { + const fetchinfo: ?FetchInfo = if (!std.mem.startsWith(u8, url.fetch, "https://ziglang.org")) + null + else fetchinfo: { var mirrors = try MirrorUrls.get(gpa, app_data_path); defer mirrors.deinit(gpa); - assert(mirrors.list.items.len > 0); + if (mirrors.list.items.len == 0) { + log.err("no zig mirrors found.", .{}); + break :fetchinfo null; + } const zig_archive_filename = url.fetch[std.mem.lastIndexOfScalar(u8, url.fetch, '/').? + 1 ..]; @@ -449,7 +454,8 @@ pub fn main() !void { const fi = try mirrors.fetchFromAny(gpa, download_path, zig_archive_filename); url.fetch = fi.archive_path; break :fetchinfo fi; - } else null; + }; + defer { if (fetchinfo) |fi| fi.deinit(gpa); } From 0d4e571a89efa3fe762389ecb2e6019689c31d60 Mon Sep 17 00:00:00 2001 From: Tobias Simetsreiter Date: Mon, 10 Nov 2025 11:05:46 +0100 Subject: [PATCH 18/18] remove deinit on arena allocated urls --- src/main.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main.zig b/src/main.zig index 7bc4533..c3a6b12 100644 --- a/src/main.zig +++ b/src/main.zig @@ -436,7 +436,6 @@ pub fn main() !void { } var url = try getVersionUrl(arena, app_data_path, semantic_version); - defer url.deinit(arena); const fetchinfo: ?FetchInfo = if (!std.mem.startsWith(u8, url.fetch, "https://ziglang.org")) null