Error when parsing `.zon` files, index out of bounds

Help
1

Here’s a little demonstration of the bug.
this is 0.15.2

const std = @import("std");

const Vector2 = struct {
    x: f32,
    y: f32,
};

const Tile = struct {
    id: usize,
    position: Vector2,
    rotation: f32 = 0,
};

pub fn main() !void {
    var debug_allocator: std.heap.DebugAllocator(.{}) = .init;
    defer _ = debug_allocator.deinit();

    const gpa = debug_allocator.allocator();

    var buffer: [2048]u8 = undefined;
    var alloc_w: std.Io.Writer.Allocating = .init(gpa);
    defer alloc_w.deinit();

    var file = try std.fs.cwd().openFile("./map.zon", .{});
    var file_reader = file.reader(&buffer);

    _ = try file_reader.interface.streamRemaining(&alloc_w.writer);

    var diag: std.zon.parse.Diagnostics = .{};
    defer diag.deinit(gpa);

    const res = std.zon.parse.fromSlice([]Tile, gpa, @ptrCast(alloc_w.written()), &diag, .{}) catch |err| {
        std.debug.print("######################\n", .{});
        std.debug.print("{f}", .{diag});
        std.debug.print("######################\n", .{});
        return err;
    };

    std.debug.print("{any}\n", .{res});
}

When I try to parse this .zon file for example.

.{ .{
    .id = 24,
    .position = .{ .x = 256, .y = 256 },
    .rotation = 0,
}, .{
    .id = 24,
    .position = .{ .x = 576, .y = 192 },
    .rotation = 0,
} }

I get this index of out bounds error.

thread 221360 panic: index out of bounds: index 165, len 164
/home/karim/.local/share/mise/installs/zig/0.15.2/lib/std/zig/tokenizer.zig:526:36: 0x11734f6 in next (std.zig)
                switch (self.buffer[self.index]) {
                                   ^
/home/karim/.local/share/mise/installs/zig/0.15.2/lib/std/zig/Ast.zig:155:37: 0x1167ec9 in parse (std.zig)
        const token = tokenizer.next();
                                    ^
/home/karim/.local/share/mise/installs/zig/0.15.2/lib/std/zon/parse.zig:278:36: 0x1160a18 in fromSlice__anon_22882 (std.zig)
    var ast = try std.zig.Ast.parse(gpa, source, .zon);
                                   ^
/home/karim/Documents/projects/code/zig-test/main.zig:32:40: 0x115eab1 in main (main.zig)
    const res = std.zon.parse.fromSlice([]Tile, gpa, @ptrCast(alloc_w.written()), &diag, .{}) catch |err| {
                                       ^
/home/karim/.local/share/mise/installs/zig/0.15.2/lib/std/start.zig:627:37: 0x115f685 in posixCallMainAndExit (std.zig)
            const result = root.main() catch |err| {
                                    ^
/home/karim/.local/share/mise/installs/zig/0.15.2/lib/std/start.zig:232:5: 0x115e6b1 in _start (std.zig)
    asm volatile (switch (native_arch) {
    ^
???:?:?: 0x0 in ??? (???)
zsh: IOT instruction (core dumped)  zig run main.zig

I’m honestly not sure if this a bug in the parser or some mistake i made…

2

The zon parser expects a null terminated string. I think the error you are running into is that reading the file into the allocator does NOT write a null to the end. You may want to try alloc_w.writeByte(0); right before passing it to the fromSlice function.

You hid this a little bit by using @ptrCast. A better option would be to get the buffer and slice it using null termination:

    const res = std.zon.parse.fromSlice([]Tile, gpa, alloc_w.written()[0.. :0], &diag, .{}) catch |err| {
        std.debug.print("######################\n", .{});
        std.debug.print("{f}", .{diag});
        std.debug.print("######################\n", .{});
        return err;
    };
3

Yeah, I didn’t say it in the post but I tired it before and still get the same:

thread 4123 panic: index out of bounds: index 166, len 165
/home/karim/.local/share/mise/installs/zig/0.15.2/lib/std/zig/tokenizer.zig:526:36: 0x1173dd6 in next (std.zig)
                switch (self.buffer[self.index]) {
                                   ^
/home/karim/.local/share/mise/installs/zig/0.15.2/lib/std/zig/Ast.zig:155:37: 0x11682a9 in parse (std.zig)
        const token = tokenizer.next();
                                    ^
/home/karim/.local/share/mise/installs/zig/0.15.2/lib/std/zon/parse.zig:278:36: 0x1160b38 in fromSlice__anon_22882 (std.zig)
    var ast = try std.zig.Ast.parse(gpa, source, .zon);
                                   ^
/home/karim/Documents/projects/code/zig-test/src/main.zig:33:40: 0x115eb65 in main (main.zig)
    const res = std.zon.parse.fromSlice([]Tile, gpa, @ptrCast(alloc_w.written()), &diag, .{}) catch |err| {
                                       ^
/home/karim/.local/share/mise/installs/zig/0.15.2/lib/std/start.zig:627:37: 0x115f7a5 in posixCallMainAndExit (std.zig)
            const result = root.main() catch |err| {
                                    ^
/home/karim/.local/share/mise/installs/zig/0.15.2/lib/std/start.zig:232:5: 0x115e6b1 in _start (std.zig)
    asm volatile (switch (native_arch) {
    ^
???:?:?: 0x0 in ??? (???)
run
└─ run exe zig_test failure
error: the following command terminated unexpectedly:
/home/karim/Documents/projects/code/zig-test/zig-out/bin/zig_test

Build Summary: 3/5 steps succeeded; 1 failed
run transitive failure
└─ run exe zig_test failure

error: the following build command failed with exit code 1:
.zig-cache/o/f1253794f4a480ef3ff8c340ea129944/build /home/karim/.local/share/mise/installs/zig/0.15.2/zig /home/karim/.local/share/mise/installs/zig/0.15.2/lib /home/karim/Documents/projects/code/zig-test .zig-cache /home/karim/.cache/zig --seed 0xfc5fb09a -Z8daecf59bbf61723 run

Funny enough this .zon file I generated it with std.zon.stringify. And when i try to parse it again it fails…

4

Oh, I forgot that when you slice the null terminated, you have to decrease the length by one.

This worked for me under 0.15.2

    try alloc_w.writer.writeByte(0);
    const buffer_slice = alloc_w.written();
    const terminated_slice = buffer_slice[0 .. buffer_slice.len - 1 :0];

    const res = std.zon.parse.fromSlice([]Tile, gpa, terminated_slice, &diag, .{}) catch |err| {
        std.debug.print("######################\n", .{});
        std.debug.print("{f}", .{diag});
        std.debug.print("######################\n", .{});
        return err;
    };
1 Like
5

Thanks, what worked. I tried the latest master and it seems like it has a better api. It didn’t require the null terminated string.

6

Yes, the new api will better fit the changes done to Writer. 0.15.X is in an odd inbetween state where we had to update the writer/reader interfaces but a lot of the code still expects the old way of doing things. It should be much more straightforward in 0.16 as it gets more integrated into the standard library.

2 Likes