How to write integration tests for CLI utilities?

Help
1

Let’s say I am writing a CLI utility, which, say, prints hello world:

// ./main.zig
const std = @import("std");

pub fn main() !void {
    try std.io.getStdOut().writeAll("hello world");
}

Now, I want to write an integration test to run the utility and check its output, something like this:

// ./tests.zig
const std = @import("std");

test "it prints hello world" {
    const exe_path: []const u8 = get_exe_path();
    const exec_result = try std.ChildProcess.exec(.{
        .allocator = std.testing.allocator,
        .argv = &.{exe_path},
    });
    try std.testing.expectEqualStrings(
        "hello world",
        exec_result.stdout,
    );
}

fn get_exe_path() []const u8 {
   ???
}

What’s the best way to get a path to exe file? That is, how should I write my build.zig such that it compiles the exe before the tests are run, and somehow injects path to the exe into tests?

I can write

// ./build.zig
const std = @import("std");

pub fn build(b: *std.build.Builder) !void {
    const hello = b.addExecutable(.{
        .name = "hello",
        .root_source_file = .{ .path = "./main.zig" },
    });

    const tests = b.addTest(.{
        .root_source_file = .{ .path = "./tests.zig" },
    });
    const run_unit_tests = b.addRunArtifact(tests);

    { // Key Point: injecting path to exe
        run_unit_tests.addArg("--exe");
        run_unit_tests.addArtifactArg(hello);
    }

    const test_tls = b.step("test", "run the tests");
    test_tls.dependOn(&hello.step);
    test_tls.dependOn(&run_unit_tests.step);
}

but this doesn’t work, as the test harness doesn’t expect the --exe argument.

Another option is adding an environmental variable, but setEnvironmentVariable requires a string, not a LazyPath or Artifact.

The third option is to install the binary into a well-known location, but that doesn’t feel right: eg, I might want to run debug and release tests concurrently, but, with a hard-coded path, I can’t.

How should this work?

2 Likes
2

The Zig build system already has you covered; std.Build.Step.Run comes with methods like expectExitCode and expectStdOutEqual that can be used to test the outputs of a program.

Here’s an example of console app that parses all arguments as integers then prints the sum, complete with a test suite that tests both success and error cases:

// main.zig

const std = @import("std");

/// Reads all arguments as integers and prints the sum to stdout.
pub fn main() !u8 {
    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
    defer std.debug.assert(gpa.deinit() == .ok);

    const allocator = gpa.allocator();

    var args = try std.process.argsWithAllocator(allocator);
    defer args.deinit();

    _ = args.skip(); // Skip the first arg (the executable name)

    var sum: i64 = 0;
    while (args.next()) |arg| {
        const term = std.fmt.parseInt(i64, arg, 10) catch {
            std.log.err("'{s}' is not a valid signed 64-bit integer", .{arg});
            return 1;
        };
        sum = std.math.add(i64, sum, term) catch {
            std.log.err("integer overflow", .{});
            return 1;
        };
    }

    try std.io.getStdOut().writer().print("{d}", .{sum});
    return 0;
}

// build.zig

const std = @import("std");

pub fn build(b: *std.Build) void {
    const target = b.standardTargetOptions(.{});
    const optimize = b.standardOptimizeOption(.{});

    const exe = b.addExecutable(.{
        .name = "sum",
        .root_source_file = .{ .path = "main.zig" },
        .target = target,
        .optimize = optimize,
    });

    const main_test_step = b.step("test", "Run integration tests");

    const test_ok = b.addRunArtifact(exe);
    test_ok.addArgs(&.{ "1", "20", "300", "9999" });
    test_ok.expectStdOutEqual("10320");
    main_test_step.dependOn(&test_ok.step);

    const test_invalid_input = b.addRunArtifact(exe);
    test_invalid_input.addArgs(&.{ "123", "abc", "xyz" });
    test_invalid_input.expectExitCode(1);
    test_invalid_input.expectStdErrEqual("error: 'abc' is not a valid signed 64-bit integer\n");
    main_test_step.dependOn(&test_invalid_input.step);

    const test_overflow = b.addRunArtifact(exe);
    test_overflow.addArgs(&.{ "1", "2", "9223372036854775807" });
    test_overflow.expectExitCode(1);
    test_overflow.expectStdErrEqual("error: integer overflow\n");
    main_test_step.dependOn(&test_overflow.step);
}

Running zig build test will compile the executable then run all three tests. Append --summary all for a more illustrative view of what the Zig build system is doing, and try changing one of the test cases to see what happens when tests fail.

9 Likes
3

That’s smart :heart: ! But I’d rather avoid writing my entire test suite in build.zig, to:

  • keep build.zig fast to compile
  • take advantage of libraries and custom code in general (eg, snapshot testing)
3 Likes
4

The simplest option (if it works, I haven’t tested it), would be to use a build option for the exe path:

    const test_options = b.addOptions();
    test_options.addOptionPath("cli_exe_path", your_cli_exe.getEmittedBin());

    const tests = b.addTest(.{
        .root_source_file = .{ .path = "./tests.zig" },
    });
    // `root_module` because of https://github.com/ziglang/zig/pull/18160
    tests.root_module.addOptions("build_options", test_options);

Usage would be:

const std = @import("std");
const build_options = @import("build_options");

test "it prints hello world" {
    const exe_path = build_options.cli_exe_path;
    // ...
}

Another way to go would be to just write a regular program that runs your tests, and then do something like:

const cli_tests = b.addExecutable(.{ ... });

const run_cli_tests = b.addRunArtifact(cli_tests);
run_cli_tests.addArtifactArg(your_cli_exe);

test_step.dependOn(&run_cli_tests.step);

This is the approach that arocc takes for some of its tests

Yet another option would be a custom test runner (std.Build.TestOptions.test_runner), and then the runner would set some shared variable to the cli exe path (this would be similar to how std.testing.allocator works).

6 Likes
5

This is how the Zig test suite works: zig/build.zig at master · ziglang/zig · GitHub

1 Like