Trouble using comptime and const for function arguments

Help
1

First post here, not sure if this belongs in here or in the Explain topic. I’ve also tried googling this issue but I haven’t found anything helpful.

I’m trying to use this blog to create a parser but I’m running into issues with comptime and const. The blog is old and made for an older version of Zig so it not working out of the box is expected but I get confused by the errors.

The code consists of a parser interface

const std = @import("std");
const Allocator = std.mem.Allocator;

pub const Error = error{
    EndOfStream,
    Utf8InvalidStartByte,
} || std.fs.File.ReadError || std.fs.File.SeekError || std.mem.Allocator.Error;

pub fn Parser(comptime Value: type, comptime Reader: type) type {
    return struct {
        const Self = @This();
        _parse: fn (self: *Self, allocator: *Allocator, src: *Reader) callconv(.Inline) Error!?Value,

        pub inline fn parse(self: *Self, allocator: *Allocator, src: *Reader) Error!?Value {
            return self._parse(self, allocator, src);
        }
    };
}

pub fn OneOf(comptime Value: type, comptime Reader: type) type {
    return struct {
        parser: Parser(Value, Reader) = .{
            ._parse = parse,
        },
        parsers: []*Parser(Value, Reader),

        const Self = @This();

        // `parsers` slice must stay alive for as long as the parser will be
        // used.
        pub fn init(parsers: []*Parser(Value, Reader)) Self {
            return Self{
                .parsers = parsers,
            };
        }

        // Caller is responsible for freeing the value, if any.
        inline fn parse(parser: *Parser(Value, Reader), allocator: *Allocator, src: *Reader) Error!?Value {
            const self: Self = @fieldParentPtr("parser", parser);
            for (self.parsers) |one_of_parser| {
                const result = try one_of_parser.parse(allocator, src);
                if (result != null) {
                    return result;
                }
            }
            return null;
        }
    };
}

And an implementation of a literal parser

const std = @import("std");
const Allocator = std.mem.Allocator;
const Parser = @import("parser.zig").Parser;
const OneOf = @import("parser.zig").OneOf;
const Error = @import("parser.zig").Error;

pub fn Literal(comptime Reader: type) type {
    return struct {
        parser: Parser([]u8, Reader) = .{
            ._parse = parse,
        },
        want: []const u8,

        const Self = @This();

        pub fn init(want: []const u8) Self {
            return Self{ .want = want };
        }

        inline fn parse(parser: *Parser([]u8, Reader), allocator: *Allocator, src: *Reader) Error!?[]u8 {
            const self: Self = @fieldParentPtr("parser", parser);
            const buf = try allocator.alloc(u8, self.want.len);
            const read = try src.reader().readAll(buf);
            if (read < self.want.len or !std.mem.eql(u8, buf, self.want)) {
                try src.seekableStream().seekBy(-@as(i64, @intCast(read)));
                allocator.free(buf);
                return null;
            }
            return buf;
        }
    };
}

test "literal" {
    const allocator = std.testing.allocator;
    var reader = std.io.fixedBufferStream("abcdef");
    const want: []const u8 = "abc";
    var literal = Literal(@TypeOf(reader)).init(want);
    const p = &literal.parser;
    const result = try p.parse(allocator, &reader);
    std.testing.expectEqualStrings(want, result.?);
    if (result) |r| {
        allocator.free(r);
    }
}

test "oneof_literal" {
    const allocator = std.testing.allocator;
    var reader = std.io.fixedBufferStream("catdogsheep");

    // Define our parser.
    var one_of = OneOf([]u8, @TypeOf(reader)).init(&.{
        &Literal(@TypeOf(reader)).init("dog").parser,
        &Literal(@TypeOf(reader)).init("sheep").parser,
        &Literal(@TypeOf(reader)).init("cat").parser,
    });
    var p = &one_of.parser;

    // Parse!
    const result = try p.parse(allocator, &reader);
    std.testing.expectEqualStrings("cat", result.?);
    if (result) |r| {
        allocator.free(r);
    }
}

Note that I’ve updated the code so that @fieldParentPtr doesn’t error.

Here I get the following compilation error:

test
└─ run test
   └─ zig test Debug native 2 errors
src/syntax.zig:38:9: error: variable of type 'syntax.Literal(io.fixed_buffer_stream.FixedBufferStream([]const u8))' must be const or comptime
    var literal = Literal(@TypeOf(reader)).init(want);
        ^~~~~~~
src/syntax.zig:9:23: note: struct requires comptime because of this field
        parser: Parser([]u8, Reader) = .{
                ~~~~~~^~~~~~~~~~~~~~
src/parser.zig:12:17: note: struct requires comptime because of this field
        _parse: fn (self: *Self, allocator: *Allocator, src: *Reader) callconv(.Inline) Error!?Value,
                ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
src/parser.zig:12:17: note: use '*const fn (comptime *parser.Parser([]u8,io.fixed_buffer_stream.FixedBufferStream([]const u8)), *mem.Allocator, *io.fixed_buffer_stream.FixedBufferStream([]const u8)) callconv(.Inline) error{EndOfSt
ream,Utf8InvalidStartByte,InputOutput,SystemResources,IsDir,OperationAborted,BrokenPipe,ConnectionResetByPeer,ConnectionTimedOut,NotOpenForReading,SocketNotConnected,WouldBlock,AccessDenied,Unexpected,Unseekable,OutOfMemory}!?[]u8
' for a function pointer type
        _parse: fn (self: *Self, allocator: *Allocator, src: *Reader) callconv(.Inline) Error!?Value,
                ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
src/syntax.zig:53:9: error: expected type '*parser.Parser([]u8,io.fixed_buffer_stream.FixedBufferStream([]const u8))', found '*const parser.Parser([]u8,io.fixed_buffer_stream.FixedBufferStream([]const u8))'
        &Literal(@TypeOf(reader)).init("dog").parser,
        ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
src/syntax.zig:53:9: note: cast discards const qualifier

If I follow the note and change the type annotation of _parse to be *const fn... I get an error saying that the assignment of ._parse = parse in the Literal parser is wrong because the first argument is not a comptime pointer to a function

Log here 
test
└─ run test
   └─ zig test Debug native 1 errors
src/syntax.zig:10:14: error: expected type '*const fn (*parser.Parser([]u8,io.fixed_buffer_stream.FixedBufferStream([]const u8)), *mem.Allocator, *io.fixed_buffer_stream.FixedBufferStream([]const u8)) callconv(.Inline) error{EndOf
Stream,Utf8InvalidStartByte,InputOutput,SystemResources,IsDir,OperationAborted,BrokenPipe,ConnectionResetByPeer,ConnectionTimedOut,NotOpenForReading,SocketNotConnected,WouldBlock,AccessDenied,Unexpected,Unseekable,OutOfMemory}!?[]
u8', found '*const fn (comptime *parser.Parser([]u8,io.fixed_buffer_stream.FixedBufferStream([]const u8)), *mem.Allocator, *io.fixed_buffer_stream.FixedBufferStream([]const u8)) callconv(.Inline) error{EndOfStream,Utf8InvalidStart
Byte,InputOutput,SystemResources,IsDir,OperationAborted,BrokenPipe,ConnectionResetByPeer,ConnectionTimedOut,NotOpenForReading,SocketNotConnected,WouldBlock,AccessDenied,Unexpected,Unseekable,OutOfMemory}!?[]u8'
            ._parse = parse,
            ~^~~~~~~~~~~~~~
src/syntax.zig:10:14: note: pointer type child 'fn (comptime *parser.Parser([]u8,io.fixed_buffer_stream.FixedBufferStream([]const u8)), *mem.Allocator, *io.fixed_buffer_stream.FixedBufferStream([]const u8)) callconv(.Inline) error
{EndOfStream,Utf8InvalidStartByte,InputOutput,SystemResources,IsDir,OperationAborted,BrokenPipe,ConnectionResetByPeer,ConnectionTimedOut,NotOpenForReading,SocketNotConnected,WouldBlock,AccessDenied,Unexpected,Unseekable,OutOfMemor
y}!?[]u8' cannot cast into pointer type child 'fn (*parser.Parser([]u8,io.fixed_buffer_stream.FixedBufferStream([]const u8)), *mem.Allocator, *io.fixed_buffer_stream.FixedBufferStream([]const u8)) callconv(.Inline) error{EndOfStre
am,Utf8InvalidStartByte,InputOutput,SystemResources,IsDir,OperationAborted,BrokenPipe,ConnectionResetByPeer,ConnectionTimedOut,NotOpenForReading,SocketNotConnected,WouldBlock,AccessDenied,Unexpected,Unseekable,OutOfMemory}!?[]u8'
src/syntax.zig:10:14: note: generic function cannot cast into a non-generic function
referenced by:
    Self: src/syntax.zig:14:22
    init: src/syntax.zig:17:20

And if I instead change the var literal = ... declaration to be const literal = ... I get an error saying that p.parse got a constant pointer where it expected a non-const pointer

Log here 
test
└─ run test
   └─ zig test Debug native 1 errors
src/syntax.zig:40:25: error: expected type '*parser.Parser([]u8,io.fixed_buffer_stream.FixedBufferStream([]const u8))', found '*const parser.Parser([]u8,io.fixed_buffer_stream.FixedBufferStream([]const u8))'
    const result = try p.parse(allocator, &reader);
                       ~^~~~~~
src/syntax.zig:40:25: note: cast discards const qualifier
src/parser.zig:14:35: note: parameter type declared here
        pub inline fn parse(self: *Self, allocator: *Allocator, src: *Reader) Error!?Value {
                                  ^~~~~

The suggestion to make the first argument a const pointer will lead to an error like the first one.

I don’t really understand at all what’s going on here and whenever I try to implement the suggestions from the compiler I just a more cryptic error. Why is it complaining about the constness when it seems to me like most things are defined at comptime? I’ve just started trying Zig so I’m sure there’s something fundamental that I’m not understanding yet.

2

Hello @ajoino welcome to ziggit :slight_smile:

  1. You need to call reader() from your stream to get a Reader.
    var stream = std.io.fixedBufferStream("abcdef");
    const reader = stream.reader();
  1. @fieldParentPtr expects a pointer to the field.
    const self: Self = @fieldParentPtr("parser", parser);

EDIT: this was not correct, parser is already a pointer to the parser field.

  1. By declaring the function type as * const fn the function pointer is available at runtime.
    _parse: *const fn (self: *Self, allocator: *Allocator, src: *Reader) callconv(.Inline) Error!?Value,
  1. Unfortunately I don’t have a solution for this. But someone else might find some fix.
    The problem is that: Parser returns a type, so the first argument is implicitly marked as comptime.
    The exact error is:
literal.zig:10:14: error: expected type '*const fn (*parser.Parser([]u8,io.GenericReader(*io.fixed_buffer_stream.FixedBufferStream([]const u8),error{},(function 'read'))), *mem.Allocator, *io.GenericReader(*io.fixed_buffer_stream.FixedBufferStream([]const u8),error{},(function 'read'))) callconv(.Inline) error{EndOfStream,Utf8InvalidStartByte,InputOutput,SystemResources,IsDir,OperationAborted,BrokenPipe,ConnectionResetByPeer,ConnectionTimedOut,NotOpenForReading,SocketNotConnected,WouldBlock,AccessDenied,Unexpected,Unseekable,OutOfMemory}!?[]u8',
                                  found '*const fn (comptime *parser.Parser([]u8,io.GenericReader(*io.fixed_buffer_stream.FixedBufferStream([]const u8),error{},(function 'read'))), *mem.Allocator, *io.GenericReader(*io.fixed_buffer_stream.FixedBufferStream([]const u8),error{},(function 'read'))) callconv(.Inline) error{EndOfStream,Utf8InvalidStartByte,InputOutput,SystemResources,IsDir,OperationAborted,BrokenPipe,ConnectionResetByPeer,ConnectionTimedOut,NotOpenForReading,SocketNotConnected,WouldBlock,AccessDenied,Unexpected,Unseekable,OutOfMemory}!?[]u8'
            ._parse = parse,
            ~^~~~~~~~~~~~~~

EDIT: an idea is to use *opaque instead of *Parser([]u8, Reader) for the parameter, and then cast to the Parser type.

1 Like
3

Thanks for the warm welcome and the helpful reply, point 4 helped me understand what the problem is.

I’m trying to implement the suggestion in your edit but I struggle to make it work. Using *opaque in the signature seems to be a syntax error (it expects {} after) and I can’t figure out how to use *anyopaque correctly here.

1 Like
4

I was able to play with your files and got the tests passing. One big note here:

Callconv

I think one of the big issues that callconv(.Inline) on the parser turns the first parameter into a comptime known Parser. It requires the implementation to be known at compile time so that the compiler can inline the function. This is probably not what you want.

By removing that callconv from the parse function type, it made it easier to move forward. You’ll also need to change inline fn parse(...) to fn parse() which is probably more acurate. You can still have pub inline fn parse(....) in the Parser type which is more of what you want here.

Diff for that:

diff parser.original.zig parser.zig
12c12
<         _parse: fn (self: *Self, allocator: *Allocator, src: *Reader) callconv(.Inline) Error!?Value,
---
>         _parse: *const fn (self: *Self, allocator: *Allocator, src: *Reader) Error!?Value,
38,39c38,39
<         inline fn parse(parser: *Parser(Value, Reader), allocator: *Allocator, src: *Reader) Error!?Value {
<             const self: Self = @fieldParentPtr("parser", parser);
---
>         fn parse(parser: *Parser(Value, Reader), allocator: *Allocator, src: *Reader) Error!?Value {
>             const self: *Self = @fieldParentPtr("parser", parser);

Another small thing to note here is that you generally see Allocator not *Allocator. The Allocator interface is small and easy to copy around, so you will usually seen it passed by copy and not by reference.

2 Likes
5

Thanks! I’ll play around with this information and hopefully I’ll get back with positive results.

6

Alright, I got something working. Thanks for the help everyone!

After moving things around I finally concluded that the problem is that I don’t understand interfaces enough and the problem here was that I was trying to create a generic interface. I want the interface to be generic to create parsers that return different value types. I decided to look around how zig does interfaces and found onto this page that outlines a way to create generic interfaces. A quick look around the stdlib (because I ran into other errors…) confirm that the blog is not alone in doing it this way.

My current version looks like this, with everything in one file.

const std = @import("std");
const Allocator = std.mem.Allocator;

pub const Error = error{
    EndOfStream,
    Utf8InvalidStartByte,
} || std.fs.File.ReadError || std.fs.File.SeekError || std.mem.Allocator.Error;

pub fn Parser(
    Pointer: type,
    ValueType: type,
    parseFn: *const fn (self: Pointer, allocator: Allocator, src: anytype) Error!?ValueType,
) type {
    return struct {
        const Self = @This();
        ptr: Pointer,
        // parseFn: *const fn (self: *anyopaque, allocator: Allocator, src: anytype) Error!?ValueType,

        pub fn parse(self: *Self, allocator: Allocator, src: anytype) Error!?ValueType {
            return parseFn(self.ptr, allocator, src);
        }
    };
}

pub fn Literal() type {
    return struct {
        const Self = @This();
        pub inline fn parser(self: *Self) Parser(*Self, []u8, parse) {
            return .{
                .ptr = self,
            };
        }
        want: []const u8,

        pub fn init(want: []const u8) Self {
            return Self{ .want = want };
        }

        fn parse(self: *Self, allocator: Allocator, src: anytype) Error!?[]u8 {
            const buf = try allocator.alloc(u8, self.want.len);
            const read = try src.reader().readAll(buf);
            if (read < self.want.len or !std.mem.eql(u8, buf, self.want)) {
                try src.seekableStream().seekBy(-@as(i64, @intCast(read)));
                allocator.free(buf);
                return null;
            }
            return buf;
        }
    };
}

test "literal" {
    var allocator = std.testing.allocator;
    var readr = std.io.fixedBufferStream("abcdef");
    // const reader = readr.reader();
    const want: []const u8 = "abc";
    var literal = Literal().init(want);
    var p = literal.parser();
    const result = try p.parse(allocator, &readr);
    try std.testing.expectEqualStrings(want, result.?);
    if (result) |r| {
        allocator.free(r);
    }
}

Still working on things, but if y’all have any input on what I could do better I’d appreciate it!

2 Likes
7

If you’re interested in looking at other parser combinators to see different ways to implement some stuff, I’d suggest checking out the mecha library. They use some tricks I found quite interesting, and might give you some ideas.

2 Likes
8

Yeah I’ve seen it and I plan to take a look later once I feel I get a better grasp of Zig. Trying to learn the hard way for now

9

A couple critiques for you.

Your Literal function isn’t really needed anymore. You can turn it into a struct and it would still have the same effect.

const Literal = struct {
    want: []const u8,
    pub fn init(want: []const u8) Literal {
        return Literal{ .want = want };
    }
    ...
}

// Then to use it
var abc_parser = Literal.init("abc")

I would also suggest pinning down what you want to accomplish with having a Parser interface. If you want to have a function that can work with anything that has the parse function, you can use anytype for that. You would just pass Literal or Number or whatever struct you have that has a parse method on it and it will work.

Your Parser should have extra methods that provide value, things that can be derived from the implementations parse method. So things like parse one, parse N, parse as many as you can in a row, etc.

1 Like
10

Thanks for the feedback!

Yeah I know Literal no longer had to be a function but I couldn’t be bothered refactoring it :person_shrugging: I think I’ve also come to the conclusion that working with anytype is probably more what I want, since the interface I created now binds the actual implementation of the parse function to the type of the interface, making it very hard to create a list of parsers.