Attempting to map a slice without allocations

Help
1

I’m getting hung up in my brain when trying to map a list from one type to another. I try to avoid using allocators when possible, and something inside me tells me I shouldn’t have to allocate here, but I’m a GC boy, so I have nothing to base that feeling on.

I’m converting a rust program to zig, and I’m trying to read bytes from a zmq socket into a tagged union. It looks like this:

pub const WorkerStatusTag = enum {
    idle,
    busy,
};

pub const WorkerStatus = union(WorkerStatusTag) {
    /// The worker is currently idle.
    idle: void,
    /// The worker is currently processing one or more requests.
    /// each u128 is a Ulid identifier for the request being handled.
    busy: []u128,
    fn parse(bytes: []const u8) !WorkerStatus {
        switch (bytes[0]) {
            0x1 => return .idle,
            // TODO: Read bytes[1..] in 16 byte chunks and return the array.
            0x2 => return .{ .busy = .{} },
            else => unreachable, // FIXME: return ParseError
        }
    }
};

The protocol is simple: either the worker sends x01 when it has no work, or it sends x02 followed by the list of ids in big endian format.

Since all the data is available in bytes, I want to just return a re-mapping of it in the structured types I’m looking for, and then copy/transfer ownership of it later when I update the struct that holds the latest status of all the workers.

Is this possible, or is it a pipe dream?

Thank you all :slight_smile:

2

You want @ptrCast. You’ll also need to specify that the u128s are 1-byte aligned using align, otherwise you’ll trigger a panic/undefined behaviour.

3

Thank you. This does appear to be the correct answer, but I’m having trouble solving the panic. I looked at the docs on alignment, I’ve gotten this far (none of my attempts work):

fn parse(bytes: []u8) !WorkerStatus {
        switch (bytes[0]) {
            0x1 => return .idle,
            0x2 => {
                std.debug.print("align: {d}\n", .{@alignOf(Client.Request.Id)});
                std.debug.print("len: {d}\n", .{bytes[1..].len});

                // Attempt #1
                const requests: []Client.Request.Id = std.mem.bytesAsSlice(u128, @as([]align(16) u8, @alignCast(@constCast(bytes[1..]))));

                // Attempt #2
                const aligned align(@alignOf(u128)) = bytes[1..];
                const requests: []Client.Request.Id = @alignCast(std.mem.bytesAsSlice(u128, aligned));

                // Attempt #3
                const requests: []Client.Request.Id = std.mem.bytesAsSlice([]u128, @as([]align(16) u8, @alignCast(bytes[1..])));

                // Attempt #4
                const requests: []align(16) u128 = @ptrCast(@alignCast(std.mem.bytesAsSlice(u128, bytes[1..])));

                // Attempt #5
                const requests = std.mem.bytesAsSlice(u128, @as([]align(16) u8, @alignCast(bytes[1..])));


                return .{ .busy = requests };
            },
            else => unreachable,
        }
    }

zig build test
test
└─ run test failure
align: 16
len: 48
thread 2387579 panic: incorrect alignment
/home/spencer/git.verticalaxion.com/vai/src/Worker.zig:77:102: 0x12b2935 in parse (root.zig)
                const requests: []Client.Request.Id = std.mem.bytesAsSlice(u128, @as([]align(16) u8, @alignCast(bytes[1..])));
                                                                                                     ^
/home/spencer/git.verticalaxion.com/vai/src/Worker.zig:200:43: 0x12b339f in test.parse (root.zig)
    const _status = try WorkerStatus.parse(&buffer);
                                          ^
/nix/store/q7ym78ggwsd3ahbcamjqafsagvsg9hvi-zig-0.15.1/lib/zig/compiler/test_runner.zig:130:29: 0x122d433 in mainServer (test_runner.zig)
                test_fn.func() catch |err| switch (err) {
                            ^
/nix/store/q7ym78ggwsd3ahbcamjqafsagvsg9hvi-zig-0.15.1/lib/zig/compiler/test_runner.zig:64:26: 0x122e6b6 in main (test_runner.zig)
        return mainServer() catch @panic("internal test runner failure");
                         ^
/nix/store/q7ym78ggwsd3ahbcamjqafsagvsg9hvi-zig-0.15.1/lib/zig/std/start.zig:618:22: 0x1228ce5 in main (std.zig)
            root.main();
                     ^
???:?:?: 0x7f68f6a2a4d7 in ??? (libc.so.6)
Unwind information for `libc.so.6:0x7f68f6a2a4d7` was not available, trace may be incomplete

???:?:?: 0x7f68f6a2a59a in ??? (libc.so.6)
???:?:?: 0x12ffe94 in ??? (???)
error: while executing test 'Worker.test.parse', the following command terminated with signal 6 (expected exited with code 0):
./.zig-cache/o/a53f244ce28a62ecf202d535b758adc4/test --cache-dir=./.zig-cache --seed=0x4efcb2b6 --listen=-

Build Summary: 8/10 steps succeeded; 1 failed; 32/32 tests passed
test transitive failure
└─ run test failure

error: the following build command failed with exit code 1:
.zig-cache/o/32db72e130a577b8042338e6aa26fb9c/build /nix/store/q7ym78ggwsd3ahbcamjqafsagvsg9hvi-zig-0.15.1/bin/zig /nix/store/q7ym78ggwsd3ahbcamjqafsagvsg9hvi-zig-0.15.1/lib/zig /home/spencer/git.verticalaxion.com/vai .zig-cache /home/spencer/.cache/zig --seed 0x4efcb2b6 -Z6a794ff4f105c0f4 test
4

Nevermind. I need to do the big/little conversion first. Thank you!

5

The align needs to be on the busy field slice:

pub const WorkerStatus = union(WorkerStatusTag) {
    /// The worker is currently idle.
    idle: void,
    /// The worker is currently processing one or more requests.
    /// each u128 is a Ulid identifier for the request being handled.
    busy: []align(1) u128,

However, the endian conversion functions in the stdlib take a []T rather than a []align(x) T , so you’ll probably run into issues there. You might need to implement that manually.

If you insist on no allocations, and can’t fix up the alignment in some other way, maybe use @ptrCast and then manually iterate over every element of the slice and convert it using busy[i] = std.mem.bigToNative(u128, busy[i])?

On another note: Does the ULID actually need to be converted, or could you just treat it as an opaque 16 bytes? So instead, busy would be declared as:

pub const WorkerStatus = union(WorkerStatusTag) {
    /// The worker is currently idle.
    idle: void,
    /// The worker is currently processing one or more requests.
    /// each u128 is a Ulid identifier for the request being handled.
    busy: []Ulid,

    const Ulid = [16]u8;

Semantically, I think it might make more sense to treat it that way. It’s not an integer, it’s a 128-bit / 16-byte unique identifier. As it’s a ULID rather than UUID, if you need to sort it, you can use std.mem.order or std.mem.lessThan to sort it lexicographically.

6

I haven’t landed yet on exactly how the Request ids should be stored.

Using busy: []align(1) u128 as you mentioned does make the test pass.

If I were to just throw the Ulid’s bytes onto the wire, would I still need to do the bigToNative conversion?

Current (working) implementation:

pub const WorkerStatus = union(WorkerStatusTag) {
    /// The worker is currently idle.
    idle: void,
    /// The worker is currently processing one or more requests.
    busy: []align(1) Client.Request.Id, // u128

    fn parse(bytes: []u8) !WorkerStatus {
        switch (bytes[0]) {
            0x1 => return .idle,
            0x2 => {
                const requests = std.mem.bytesAsSlice(u128, bytes[1..]);
                std.debug.print("requests: {}\n", .{@TypeOf(requests)});
                for (requests, 0..) |request, i| {
                    requests[i] = std.mem.bigToNative(u128, request);
                }

                return .{ .busy = requests };
            },
            else => unreachable,
        }
    }
};

test "parse" {
    const Ulid = @import("./ulid/Ulid.zig");

    var buffer: [49]u8 = undefined;
    buffer[0] = 0x2;

    const input: [3]Ulid = .{ Ulid.new(), Ulid.new(), Ulid.new() };

    var offset: usize = 1;
    for (input) |u| {
        const start = offset;
        const end = offset + 16;

        var num_buf: [16]u8 = undefined;
        const num = u.toU128();
        std.mem.writeInt(u128, &num_buf, num, .big);
        @memcpy(buffer[start..end], &num_buf);

        offset += 16;
    }

    var buf: [16]u8 = undefined;
    std.mem.writeInt(u128, &buf, input[0].toU128(), .big);
    try std.testing.expectEqualSlices(u8, &buf, buffer[1..17]);

    const _status = try WorkerStatus.parse(&buffer);

    try std.testing.expectEqual(WorkerStatusTag.busy, @as(WorkerStatusTag, _status));
    try std.testing.expectEqual(input[0].toU128(), _status.busy[0]);
    try std.testing.expectEqual(input[1].toU128(), _status.busy[1]);
    try std.testing.expectEqual(input[2].toU128(), _status.busy[2]);
}