Does anyone have an example of emitting a tar file as a build artifact, for example emitting a tar of two executables?
You could take a look at the build.zig changes in this pr feat: add macOS support by tensorush · Pull Request #70 · andrewrk/poop · GitHub. Haven’t tested it personally but that is the only example that I know of.
I see it uses const release_exe_archive = b.addSystemCommand(&.{ "tar", "-cJf" });
I’m going to attempt making a tool step, as outlined in Zig Build System ⚡ Zig Programming Language
The tool step will accept b.LazyPaths as file arguments and write the tar to the output file argument
This is going to be so much easier when CLI argument parser comes to std.
Check out this bad boy, we are absolutely COOKING.
jeff@jeff-debian:~/repos/oci$ tree zig-out/
zig-out/
└── bin
├── blob
│ ├── my-binary
│ └── my-binary2
├── blob.tar
└── hello_world
3 directories, 4 files
(I expanded the tar)
jeff@jeff-debian:~/repos/oci$ diff ./zig-out/bin/blob/my-binary ./zig-out/bin/hello_world -s
Files ./zig-out/bin/blob/my-binary and ./zig-out/bin/hello_world are identical
jeff@jeff-debian:~/repos/oci$ diff ./zig-out/bin/blob/my-binary2 ./zig-out/bin/hello_world -s
Files ./zig-out/bin/blob/my-binary2 and ./zig-out/bin/hello_world are identical
jeff@jeff-debian:~/repos/oci$ chmod +x ./zig-out/bin/blob/my-binary2
jeff@jeff-debian:~/repos/oci$ ./zig-out/bin/blob/my-binary2
hello world
Gonna need to make it executable though…
create_layer.zig
/// This is the "tool" used to create a layer.
///
/// It accepts command-line arguments provided by
/// oci.build.createLayer.
///
/// The first CLI argument is the output path to write
/// the tar file.
///
/// The remaining CLI arguments are file path pairs, where
/// the first path in the pair is the source file and the second path in the pair is the path to place the source file in the layer.
///
/// This tool step needs to be compatabile with all build system hosts, which includes (at least):
/// 1. windows
/// 2. linux
/// 3. wasm (future)
///
/// cli args (they must be in this order)
/// positional:
/// - []const u8 : output path for the layer tar
/// - []const []const u8 : (pairs of input file path and dest in tar)
///
/// This is basically the same as what `tar -cf` does.
const create_layer = @This();
const std = @import("std");
pub fn main() !void {
var gpa = std.heap.DebugAllocator(.{}).init;
defer _ = gpa.deinit();
var arena_instance = std.heap.ArenaAllocator.init(gpa.allocator());
defer arena_instance.deinit();
const arena = arena_instance.allocator();
const args = try std.process.argsAlloc(arena);
if (!(args.len > 3)) {
std.log.err("invalid number of arguments: {}", .{args.len});
return error.InvalidArguments;
}
if (args.len % 2 == 1) {
std.log.err("invalid number of arguments: {}, expected even", .{args.len});
for (args, 0..) |arg, i| {
std.log.err(" arg[{}]: {s}", .{ i, arg });
}
return error.InvalidArguments;
}
// NOTE: first arg is ignored since its just this exe
const output_path = args[1];
const output_file = try std.fs.createFileAbsolute(output_path, .{});
defer output_file.close();
const output_buffer: []u8 = try arena.alloc(u8, 4096);
var output_file_writer = std.fs.File.Writer.init(output_file, output_buffer);
const output_file_writer_interface: *std.Io.Writer = &output_file_writer.interface;
// NOTE: mtime = 0 provides build reproducibility
var tar_writer = std.tar.Writer{
.mtime_now = 0,
.prefix = "",
.underlying_writer = output_file_writer_interface,
};
const reader_buffer: []u8 = try arena.alloc(u8, 4096);
for (args[2..], 2..) |input_file_path, i| {
if (i % 2 == 1) continue;
const output_file_path = args[i + 1];
const input_file = try std.fs.cwd().openFile(input_file_path, .{ .mode = .read_only });
defer input_file.close();
const input_file_size: u64 = (try input_file.stat()).size;
var input_file_reader = input_file.reader(reader_buffer);
const input_file_reader_interface: *std.Io.Reader = &input_file_reader.interface;
try tar_writer.writeFileStream(
output_file_path,
input_file_size,
input_file_reader_interface,
.{ .mtime = 0, .mode = 0 },
);
}
// NOTE: zig std does not recommend calling this, so we won't.
// It ends the tar archive with two zero blocks. Which is a waste of space.
// tar_writer.finishPedantically();
}
pub const Layer = struct {
blob: std.Build.LazyPath,
step: std.Build.Step,
};
pub const Copy = struct {
src: std.Build.LazyPath,
dest: []const u8,
};
pub fn createLayer(b: *std.Build, sources: []const Copy) Layer {
const tar_cf_tool = b.addExecutable(.{
.name = "create_layer",
.root_module = b.createModule(.{
.root_source_file = b.path("src/build/create_layer.zig"),
.target = b.graph.host,
}),
});
const run = b.addRunArtifact(tar_cf_tool);
const blob = run.addOutputFileArg("blob.tar");
for (sources) |copy| {
run.addFileArg(copy.src);
run.addArg(std.mem.trimStart(u8, copy.dest, "/"));
}
return Layer{ .blob = blob, .step = run.step };
}
build.zig
// test, hello world
const hello_world_mod = b.createModule(.{
.root_source_file = b.path("test/hello_world.zig"),
.target = target,
.optimize = optimize,
});
const hello_world_exe = b.addExecutable(.{
.name = "hello_world",
.root_module = hello_world_mod,
});
const hello_world_install = b.addInstallArtifact(hello_world_exe, .{});
test_step.dependOn(&hello_world_install.step);
const hello_world_run = b.addRunArtifact(hello_world_exe);
const hello_world_run_step = b.step("run", "run hello world exe");
hello_world_run_step.dependOn(&hello_world_run.step);
// try out createLayer
const layer = createLayer(
b,
&.{
.{ .src = hello_world_exe.getEmittedBin(), .dest = "/my-binary" },
.{ .src = hello_world_exe.getEmittedBin(), .dest = "/my-binary2" },
},
);
const create_layer_step = b.step("create-layer", "try out create layer");
const install_layer = b.addInstallBinFile(layer.blob, "blob.tar");
create_layer_step.dependOn(&install_layer.step);
1 Like
This allows me to set the file perms (to allow executable). Also I forgot to flush. Also this is for current zig master.
/// This is the "tool" used to create a layer.
///
/// It accepts command-line arguments provided by
/// oci.build.createLayer.
///
/// The first CLI argument is the output path to write
/// the tar file.
///
/// The remaining CLI arguments are triplets of file path, file path, octal perms, where
/// the first path is the source file and the second pathis the path to place the source file in the layer. Example perm is 775.
///
/// This tool step needs to be compatabile with all build system hosts, which includes (at least):
/// 1. windows
/// 2. linux
/// 3. wasm (future)
///
/// cli args (they must be in this order)
/// positional:
/// - []const u8 : output path for the layer tar
/// - []const []const u8 : (triplets of input file path, dest in tar, and file mode in octal)
///
///
/// This is basically the same as what `tar -cf` does.
const create_layer = @This();
const std = @import("std");
pub fn main() !void {
var io_impl = std.Io.Threaded.init_single_threaded;
const io = io_impl.io();
var gpa = std.heap.DebugAllocator(.{}).init;
defer _ = gpa.deinit();
var arena_instance = std.heap.ArenaAllocator.init(gpa.allocator());
defer arena_instance.deinit();
const arena = arena_instance.allocator();
const args = try std.process.argsAlloc(arena);
if (!(args.len > 3)) {
std.log.err("invalid number of arguments: {}", .{args.len});
return error.InvalidArguments;
}
if (args.len % 3 == 1) {
std.log.err("invalid number of arguments: {}", .{args.len});
for (args, 0..) |arg, i| {
std.log.err(" arg[{}]: {s}", .{ i, arg });
}
return error.InvalidArguments;
}
// NOTE: first arg is ignored since its just this exe
const output_path = args[1];
const output_file = try std.fs.createFileAbsolute(output_path, .{});
defer output_file.close();
const output_buffer: []u8 = try arena.alloc(u8, 4096);
var output_file_writer = std.fs.File.Writer.init(output_file, output_buffer);
const output_file_writer_interface: *std.Io.Writer = &output_file_writer.interface;
// NOTE: mtime = 0 provides build reproducibility
var tar_writer = std.tar.Writer{
.prefix = "",
.underlying_writer = output_file_writer_interface,
};
const reader_buffer: []u8 = try arena.alloc(u8, 4096);
for (args[2..], 2..) |input_file_path, i| {
if (i % 3 != 2) continue;
const output_file_path = args[i + 1];
const output_file_mode = std.fmt.parseInt(u32, args[i + 2], 8) catch {
std.log.err("invalid mode: {s}", .{args[i + 2]});
return error.InvalidMode;
};
const input_file = try std.fs.cwd().openFile(input_file_path, .{ .mode = .read_only });
defer input_file.close();
const input_file_size: u64 = (try input_file.stat()).size;
var input_file_reader = input_file.reader(io, reader_buffer);
const input_file_reader_interface: *std.Io.Reader = &input_file_reader.interface;
try tar_writer.writeFileStream(
output_file_path,
input_file_size,
input_file_reader_interface,
.{ .mtime = 0, .mode = output_file_mode },
);
}
try tar_writer.finishPedantically();
try output_file_writer_interface.flush();
}
pub const Layer = struct {
blob: std.Build.LazyPath,
step: std.Build.Step,
};
pub const Copy = struct {
src: std.Build.LazyPath,
dest: []const u8,
mode: u32,
};
pub fn createLayer(b: *std.Build, sources: []const Copy) Layer {
const tar_cf_tool = b.addExecutable(.{
.name = "create_layer",
.root_module = b.createModule(.{
.root_source_file = b.path("src/build/create_layer.zig"),
.target = b.graph.host,
}),
});
const run = b.addRunArtifact(tar_cf_tool);
const blob = run.addOutputFileArg("blob.tar");
for (sources) |copy| {
run.addFileArg(copy.src);
run.addArg(std.mem.trimStart(u8, copy.dest, "/"));
const octal_mode: []const u8 = std.fmt.allocPrint(b.allocator, "{o}", .{copy.mode}) catch @panic("oom");
run.addArg(octal_mode);
}
return Layer{ .blob = blob, .step = run.step };
}
// test, hello world
const hello_world_mod = b.createModule(.{
.root_source_file = b.path("test/hello_world.zig"),
.target = target,
.optimize = optimize,
});
const hello_world_exe = b.addExecutable(.{
.name = "hello_world",
.root_module = hello_world_mod,
});
const hello_world_install = b.addInstallArtifact(hello_world_exe, .{});
test_step.dependOn(&hello_world_install.step);
const hello_world_run = b.addRunArtifact(hello_world_exe);
const hello_world_run_step = b.step("run", "run hello world exe");
hello_world_run_step.dependOn(&hello_world_run.step);
// try out createLayer
const layer = createLayer(
b,
&.{
.{ .src = hello_world_exe.getEmittedBin(), .dest = "/my-binary", .mode = 0o775 },
.{ .src = hello_world_exe.getEmittedBin(), .dest = "/my-binary2", .mode = 0o775 },
},
);
const create_layer_step = b.step("create-layer", "try out create layer");
const install_layer = b.addInstallBinFile(layer.blob, "blob.tar");
create_layer_step.dependOn(&install_layer.step);
8 Likes
And here I was perfectly happy with my “calling out to system tar” solution. 
That’s the way to go, we similarly cook Zip files for TigerBeetle:
2 Likes