Idiomatic unit testing suggestion: avoid `inline for`

Explain
1

I see a lot of people give in to the temptation to abuse inline for and write tests like this (real example from the standard library):

test "twos complement limit set" {
    const test_types = [_]type{
        u64,
        i64,
        u1,
        i1,
        u0,
        i0,
        u65,
        i65,
    };

    inline for (test_types) |T| {
        const int_info = @typeInfo(T).Int;

        var a = try Managed.init(testing.allocator);
        defer a.deinit();

        try a.setTwosCompIntLimit(.max, int_info.signedness, int_info.bits);
        const max: T = maxInt(T);
        try testing.expect(max == try a.to(T));

        try a.setTwosCompIntLimit(.min, int_info.signedness, int_info.bits);
        const min: T = minInt(T);
        try testing.expect(min == try a.to(T));
    }
}

When a test failure happens, it looks like this:

Test [2213/2751] math.big.int_test.test.twos complement limit set... Segmentation fault at address 0x0
/home/andy/src/zig/lib/libc/musl/arch/i386/atomic_arch.h:83:2: 0x2187b56 in __stack_chk_fail (/home/andy/src/zig/lib/libc/musl/src/env/__stack_chk_fail.c)
 __asm__ __volatile__( "hlt" : : : "memory" );
 ^
Unwind error at address `exe:0x2187b56` (error.AddressOutOfRange), trace may be incomplete

/home/andy/src/zig/lib/std/math/big/int.zig:2761:33: 0x1b691b0 in to__anon_55641 (test)
        return self.toConst().to(T);
                                ^
/home/andy/src/zig/lib/std/math/big/int_test.zig:310:43: 0x1b68b8a in test.twos complement limit set (test)
        try testing.expect(max == try a.to(T));
                                          ^
/home/andy/src/zig/lib/compiler/test_runner.zig:158:25: 0xb89d2b in mainTerminal (test)
        if (test_fn.func()) |_| {
                        ^
/home/andy/src/zig/lib/compiler/test_runner.zig:35:28: 0xb37685 in main (test)
        return mainTerminal();
                           ^
/home/andy/src/zig/lib/std/start.zig:501:22: 0xb3710f in main (test)
            root.main();
                     ^
/home/andy/src/zig/lib/libc/musl/src/env/__libc_start_main.c:95:7: 0x2187890 in libc_start_main_stage2 (/home/andy/src/zig/lib/libc/musl/src/env/__libc_start_main.c)
 exit(main(argc, argv, envp));
      ^
???:?:?: 0xff903d64 in ??? (???)
error: the following test command crashed:
/home/andy/src/zig/zig-cache/o/a15660423f74135612b38da4d3379840/test

Instead, I suggest to make a function call for each type:

test "twos complement limit set" {
    try testTwosComplementLimit(u64);
    try testTwosComplementLimit(i64);
    try testTwosComplementLimit(u1);
    try testTwosComplementLimit(i1);
    try testTwosComplementLimit(u0);
    try testTwosComplementLimit(i0);
    try testTwosComplementLimit(u65);
    try testTwosComplementLimit(i65);
}

fn testTwosComplementLimit(comptime T: type) !void {
    const int_info = @typeInfo(T).Int;

    var a = try Managed.init(testing.allocator);
    defer a.deinit();

    try a.setTwosCompIntLimit(.max, int_info.signedness, int_info.bits);
    const max: T = maxInt(T);
    try testing.expect(max == try a.to(T));

    try a.setTwosCompIntLimit(.min, int_info.signedness, int_info.bits);
    const min: T = minInt(T);
    try testing.expect(min == try a.to(T));
}

Now the same test failure looks like this:

Test [2213/2751] math.big.int_test.test.twos complement limit set... Segmentation fault at address 0x0
/home/andy/src/zig/lib/libc/musl/arch/i386/atomic_arch.h:83:2: 0x21879b6 in __stack_chk_fail (/home/andy/src/zig/lib/libc/musl/src/env/__stack_chk_fail.c)
 __asm__ __volatile__( "hlt" : : : "memory" );
 ^
Unwind error at address `exe:0x21879b6` (error.AddressOutOfRange), trace may be incomplete

/home/andy/src/zig/lib/std/math/big/int.zig:2761:33: 0x1b68c00 in to__anon_55649 (test)
        return self.toConst().to(T);
                                ^
/home/andy/src/zig/lib/std/math/big/int_test.zig:309:39: 0x1b689b4 in testTwosComplementLimit__anon_55648 (test)
    try testing.expect(max == try a.to(T));
                                      ^
/home/andy/src/zig/lib/std/math/big/int_test.zig:297:32: 0x1b690ff in test.twos complement limit set (test)
    try testTwosComplementLimit(u65);
                               ^
/home/andy/src/zig/lib/compiler/test_runner.zig:158:25: 0xb89ebb in mainTerminal (test)
        if (test_fn.func()) |_| {
                        ^
/home/andy/src/zig/lib/compiler/test_runner.zig:35:28: 0xb37815 in main (test)
        return mainTerminal();
                           ^
/home/andy/src/zig/lib/std/start.zig:501:22: 0xb3729f in main (test)
            root.main();
                     ^
/home/andy/src/zig/lib/libc/musl/src/env/__libc_start_main.c:95:7: 0x21876f0 in libc_start_main_stage2 (/home/andy/src/zig/lib/libc/musl/src/env/__libc_start_main.c)
 exit(main(argc, argv, envp));
      ^
???:?:?: 0xffbccd64 in ??? (???)
error: the following test command crashed:
/home/andy/src/zig/zig-cache/o/a15660423f74135612b38da4d3379840/test

Do you see the difference? We get a super helpful clue now, which is that it crashed on the u65 case.

And it’s fewer lines of code!

 lib/std/math/big/int_test.zig | 40 +++++++++++++++++++---------------------
 1 file changed, 19 insertions(+), 21 deletions(-)
26 Likes
2

It’s also important to point out that certain things will be hidden by loops that should be tested. I just came across an example of this recently (I’ll condense it here to keep the point clear):

Let’s say you’re testing an iterator. It has a next function that should return one element per call and eventually iterate over every element in a string:

const string: []const u8 = "abcdefg";

var itr = Iterator.init(string);

for (string) |c| { 
   const v = itr.next() orelse unreachable;
   try std.testing.expectEqual(c, v);
}

Now, here we can see that the iterator is supposed to match every character in the string. However, the iterator should actually be spent by the time this loop is over… but this loop doesn’t test for that. In fact, the iterator may fail if we call next again, but we can’t see that because of how we structured our test.

Loop tests can fail in other ways, too. In the next example, there’s no guarantee that we matched each element:

const string: []const u8 = "abcdefg";

var itr = Iterator.init(string);

var i: usize = 0;

while (itr.next()) |v| { 
   try std.testing.expectEqual(string[i], v);
   i += 1;
}

Long story short, I agree with the premise here. When possible, it’s best to be explicit and spell it out before moving on to incorporate more language features. Essentially, don’t try to be clever when testing things.

6 Likes
3

any reason why people should not expect the inline variant error to display the resolved value of T?

3 Likes