How to write integration tests for CLI utilities?

Help
2

The Zig build system already has you covered; std.Build.Step.Run comes with methods like expectExitCode and expectStdOutEqual that can be used to test the outputs of a program.

Here’s an example of console app that parses all arguments as integers then prints the sum, complete with a test suite that tests both success and error cases:

// main.zig

const std = @import("std");

/// Reads all arguments as integers and prints the sum to stdout.
pub fn main() !u8 {
    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
    defer std.debug.assert(gpa.deinit() == .ok);

    const allocator = gpa.allocator();

    var args = try std.process.argsWithAllocator(allocator);
    defer args.deinit();

    _ = args.skip(); // Skip the first arg (the executable name)

    var sum: i64 = 0;
    while (args.next()) |arg| {
        const term = std.fmt.parseInt(i64, arg, 10) catch {
            std.log.err("'{s}' is not a valid signed 64-bit integer", .{arg});
            return 1;
        };
        sum = std.math.add(i64, sum, term) catch {
            std.log.err("integer overflow", .{});
            return 1;
        };
    }

    try std.io.getStdOut().writer().print("{d}", .{sum});
    return 0;
}

// build.zig

const std = @import("std");

pub fn build(b: *std.Build) void {
    const target = b.standardTargetOptions(.{});
    const optimize = b.standardOptimizeOption(.{});

    const exe = b.addExecutable(.{
        .name = "sum",
        .root_source_file = .{ .path = "main.zig" },
        .target = target,
        .optimize = optimize,
    });

    const main_test_step = b.step("test", "Run integration tests");

    const test_ok = b.addRunArtifact(exe);
    test_ok.addArgs(&.{ "1", "20", "300", "9999" });
    test_ok.expectStdOutEqual("10320");
    main_test_step.dependOn(&test_ok.step);

    const test_invalid_input = b.addRunArtifact(exe);
    test_invalid_input.addArgs(&.{ "123", "abc", "xyz" });
    test_invalid_input.expectExitCode(1);
    test_invalid_input.expectStdErrEqual("error: 'abc' is not a valid signed 64-bit integer\n");
    main_test_step.dependOn(&test_invalid_input.step);

    const test_overflow = b.addRunArtifact(exe);
    test_overflow.addArgs(&.{ "1", "2", "9223372036854775807" });
    test_overflow.expectExitCode(1);
    test_overflow.expectStdErrEqual("error: integer overflow\n");
    main_test_step.dependOn(&test_overflow.step);
}

Running zig build test will compile the executable then run all three tests. Append --summary all for a more illustrative view of what the Zig build system is doing, and try changing one of the test cases to see what happens when tests fail.

11 Likes