Improving the API for a trailing data structure wrapper

I’m trying to improve the API for my “extra data” system: extra_data.zig · GitHub

Here is a practical example of how the current API is used in my renderer:

pub const RenderBundle = struct {

    // nil indicates unindexed drawing
    index_buffer: IndexBufferHandle = IndexBufferHandle.nil,
    index_buffer_offset: i32 = 0,
    material_instance: MaterialInstanceHandle = MaterialInstanceHandle.nil,
    base_element: u32 = 0,
    num_elements: u32,
    extra: Extra.Lengths align(16),

    pub const Extra = extra_data.ExtraData(@This(), .{
        .vertex_bindings = .{ VertexBinding, u3 },

        // List of material instance passes that use instance vertex attributes that are part of this bundle.
        .included_passes = .{ IncludedPass, u3 },

        // Instance vertex attribute data for all the include_passes in this bundle.
        .instance_attributes_bytes = .{ u8, u16 },
    }, .{
        // Drawing will append instance vertex data, so they it needs to be at the
        // end in order to use the fast copy path in ExtraData.dupe
        .sort_by_alignment = false,
    });

    pub const VertexBinding = struct {
        buffer: VertexBufferHandle,
        offset: i32 = 0,
    };

    pub const IncludedPass = struct {
        index: u3,
        instance_attributes_offset: u16,
    };

    ...

Creating an instance looks like this:

    const render_bundle = try RenderBundle.Extra.create(renderer.allocator, .{
      .num_vertex_bindings = 1,
    });

    render_bundle.* = .{
      .num_elements = self.extra.num_indices,
      .extra = render_bundle.extra,
    };

    const bundle_accessor = RenderBundle.Extra.accessor(render_bundle);
    bundle_accessor.vertex_bindings[0].buffer = render.VertexBufferHandle.nil;

In this case, this data structure is used as a template, which is duplicated into a buffer (passing a new set of lengths) when render commands for the frame are appended:

    var lengths = bundle.extra;
    lengths.num_included_passes = 0;
    lengths.num_instance_attributes_bytes = 0;
    for (passes) |pass| {
        lengths.num_included_passes += if (pass.instance_attributes_len > 0) 1 else 0;
        lengths.num_instance_attributes_bytes += pass.instance_attributes_len;
    }

    const bundle_ptr = try RenderBundle.Extra.dupe(self.allocator.allocator(), lengths, bundle);
    const bundle_accessor = RenderBundle.Extra.accessor(bundle_ptr);
    const new_commands = self.dispatch.render_commands.addManyAsSliceAssumeCapacity(passes.len);
    bundle_ptr.material_instance = material_instance;

    var instance_attribute_offset: u16 = 0;
    var included_passes_ix: usize = 0;
    for (new_commands, passes) |*command, pass| {
        if (pass.instance_attributes_len > 0) {
            bundle_accessor.included_passes[included_passes_ix] = .{
                .index = pass.index,
                .instance_attributes_offset = instance_attribute_offset,
            };

            included_passes_ix += 1;
        }

    ...

Separately, I was working on a serialization system (referencing Serialization For Games | Joren's) and I realized that the relative data structures mentioned in the article were somewhat similar to ExtraData. Taking a simpler example:

const Derived = extern struct {
    width: u16,
    height: u16,
    settings: Settings,
    trailing: Trailing.Lengths,

    const Trailing = ExtraData(@This(), .{
        .pixel_data = .{ u8, u32 },
    });
};

Since the header struct is extern, then I think the entire allocation (accessible via ie. Derived.Trailing.allocation(ptr) could be serialized directly - since access always happens relative to the pointer the header, accessor() can build the slices to all the trailing data arrays. This is essentially the same concept as the Relative Array mentioned in the article above.

What I’m trying to improve is the definition of the header, for example:

const Derived = extern struct {
    width: u16,
    height: u16,
    settings: Settings,
    pixel_data: RelativeSlice(u8, u32),
};

Finally this gets to my question: is it possible to implement an analog of ExtraData.create / ExtraData.allocation if the header structs were defined like this?

The part I’m not sure is possible to do at comptime is to enumerate all the fields that are “some kind of” RelativeSlice. ie. iterate all the fields of the struct in an inline for, and specifically know if one of the field types came from a call to RelativeSlice - I don’t think knowing the arguments is important as if you knew it was a RelativeSlice generic instantiation, you could just call methods on that instance to determine the size, etc.

I know it could be done by using ie. meta.hasFn(field.type, "isRelative"), but this seems fragile in general due to potential name collisions.

TDLR: is there a way to know at comptime if a type was returned from a specific function? Given a type instance, can I know if it came from a call to RelativeSlice?

This kind of feels like a kludge, and you said knowing the arguments isn’t as important as knowing if the type came from RelativeSlice … but if you can determine the arguments, you can call RelativeSlice again and compare the returned type to the given type. For example:

fn RelativeSlice(comptime ValueType: type, comptime LengthType: type) type {
    return struct {
        pub const _ValueType = ValueType;
        pub const _LengthType = LengthType;
    };
}

fn isRelativeSliceType(comptime T: type) bool {
    // TODO: Improve error handling.
    const ValueType = T._ValueType;
    const LengthType = T._LengthType;
    return T == RelativeSlice(ValueType, LengthType);
}

Note that an unrelated struct type containing variables named _ValueType and _LengthType won’t fool isRelativeSliceType. If you don’t want extra “secret” variables, you can determine the arguments however you’d like, for instance, by looking at the return type of some function declared in RelativeSlice.

1 Like

That’s a cool idea! The “secret” variables are fine I think as they are generated anyway - and you could use @hasDecl there to discover if _ValueType and _LengthType are there or not.

This idea worked quite well! The final result: relative.zig · GitHub

An example of the usage from another test case (where these structures are being serialized):

    const F = extern struct {
        x: u32 align(8),
        y: u32,
        z: relative.Pointer(u64),
    };

    const Foo = extern struct {
        a: u32,
        b: i16,
        c: relative.Slice(f32, u8),
        d: u64,
        e: relative.Pointer(u64),
        f: relative.Slice(F, u16),
    };

    const foo_in = try relative.alloc(testing.allocator, Foo, .{
        .a = 0xaabbccdd,
        .b = -1,
        .c = .{ .len = 3 },
        .d = 6,
        .e = .{},
        .f = .{ .len = 2 },
    });
    const buf = relative.allocation(Foo, foo_in);
    defer testing.allocator.free(buf);

    const c_expected: [3]f32 = .{ 3.0, 4.0, 5.0 };
    const e_expected: u64 = 7;
    const f_expected: [2]F = .{
        .{ .x = 8, .y = 9, .z = .{} },
        .{ .x = 12, .y = 13, .z = .{} },
    };
    const f_expected_z_values: [2]?u64 = .{ 10, null };

    @memcpy(foo_in.c.slice(), c_expected[0..]);
    foo_in.e.ptr().?.* = e_expected;
    for (foo_in.f.slice(), f_expected[0..], f_expected_z_values[0..]) |*f, expected, expected_z| {
        f.x = expected.x;
        f.y = expected.y;
        if (expected_z) |v| {
            f.z.ptr().?.* = v;
        } else {
            f.z.set(null);
        }
    }

    {
        const serialized = try serializeMappableAlloc(testing.allocator, Foo, foo_in, native_endian);
        defer testing.allocator.free(serialized);

        const foo_mapped = std.mem.bytesAsValue(Foo, serialized);
        try testing.expectEqual(foo_in.a, foo_mapped.a);
        try testing.expectEqual(foo_in.b, foo_mapped.b);
        try testing.expectEqualSlices(f32, &c_expected, foo_mapped.c.slice());
        try testing.expectEqual(foo_in.d, foo_mapped.d);
        try testing.expectEqual(e_expected, foo_mapped.e.ptr().?.*);

        for (f_expected, f_expected_z_values, foo_mapped.f.slice()) |expected, expected_z, *actual| {
            try testing.expectEqual(expected.x, actual.x);
            try testing.expectEqual(expected.y, actual.y);

            const actual_z = if (actual.z.ptr()) |p| p.* else null;
            try testing.expectEqual(expected_z, actual_z);
        }
    }
1 Like