0.15.1 Reader/Writer

Help
,
1

I’m having difficulties in understanding how the new API should be used. I didn’t understand much the old one either tbh. Is there somewhere a guide on this subject (valid for 0.15.1)? Or at least some example?

For example I was reading a file like this

    var buffered = std.io.bufferedReader(file.reader());
    const reader = buffered.reader();

    while (try reader.readUntilDelimiterOrEofAlloc(e.allocator, '\n', maxUsize)) |line| {

What would be now the way to read a file line by line with a buffered reader? I’m trying this but it doesn’t work:

/// Read all lines from file.
fn readLines(e: *Editor, file: std.fs.File) !void {
    var buf: [1024]u8 = undefined;
    const reader = file.reader(&buf);

    while (try reader.takeDelimiterExclusive('\n')) |line| {
        // stuff
    }
}

Thanks in advance

1 Like
2

same problem here. but mainly because vscode, zls or codeldb stopped working properly and cannot find the methods i need :slight_smile:

1 Like
3

This compiles

/// Read all lines from file.
fn readLines(e: *Editor, file: std.fs.File) !void {
    var buf: [1024 * 1024]u8 = undefined;
    var reader = file.reader(&buf).interface;

    while (reader.takeDelimiterExclusive('\n')) |line| {
        try e.insertRow(e.buffer.rows.items.len, line);
    } else |err| if (err != error.EndOfStream) return err;
}

called like this:

    // read lines if the file could be opened
    const file = std.fs.cwd().openFile(path, .{ .mode = .read_only });
    if (file) |f| {
        defer f.close();
        try e.readLines(f);
    }

but when I try to read a file I get

Summary 
thread 41462 panic: switch on corrupt value
/home/gianmaria/.local/versions/zig/0.15.1/lib/std/fs/File.zig:1315:17: 0x1195aab in stream (std.zig)
        switch (r.mode) {
                ^
/home/gianmaria/.local/versions/zig/0.15.1/lib/std/Io/Reader.zig:774:34: 0x117ee76 in peekDelimiterInclusive (std.zig)
        const n = r.vtable.stream(r, &writer, .limited(end_cap.len)) catch |err| switch (err) {
                                 ^
/home/gianmaria/.local/versions/zig/0.15.1/lib/std/Io/Reader.zig:804:44: 0x116e993 in takeDelimiterExclusive (std.zig)
    const result = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) {
                                           ^
/home/gianmaria/Progetti/zig/kilo/src/Editor.zig:162:41: 0x1153233 in readLines (main.zig)
    while (reader.takeDelimiterExclusive('\n')) |line| {
                                        ^
/home/gianmaria/Progetti/zig/kilo/src/Editor.zig:94:24: 0x11540e6 in openFile (main.zig)
        try e.readLines(f);
                       ^
/home/gianmaria/Progetti/zig/kilo/src/Editor.zig:64:23: 0x116ae5e in startUp (main.zig)
        try e.openFile(name);
                      ^
/home/gianmaria/Progetti/zig/kilo/src/main.zig:25:18: 0x116b61e in main (main.zig)
    try e.startUp(args.next()); // possible file to open
                 ^
/home/gianmaria/.local/versions/zig/0.15.1/lib/std/start.zig:627:37: 0x116c0d9 in posixCallMainAndExit (std.zig)
            const result = root.main() catch |err| {
                                    ^
/home/gianmaria/.local/versions/zig/0.15.1/lib/std/start.zig:232:5: 0x114f811 in _start (std.zig)
    asm volatile (switch (native_arch) {
    ^
???:?:?: 0x0 in ??? (???)
Aborted (core dumped)

If I reduce the buffer size to 1024 I get instead

Summary 
error: ReadFailed
/home/gianmaria/Progetti/zig/kilo/src/Editor.zig:164:48: 0x1153443 in readLines (main.zig)
    } else |err| if (err != error.EndOfStream) return err;
                                               ^
/home/gianmaria/Progetti/zig/kilo/src/Editor.zig:94:9: 0x115415e in openFile (main.zig)
        try e.readLines(f);
        ^
/home/gianmaria/Progetti/zig/kilo/src/Editor.zig:64:9: 0x116ae76 in startUp (main.zig)
        try e.openFile(name);
        ^
/home/gianmaria/Progetti/zig/kilo/src/main.zig:25:5: 0x116b6f3 in main (main.zig)
    try e.startUp(args.next()); // possible file to open
    ^
4

maybe initialization of the reader helps?

var reader: std.fs.File.Reader = .init(file, &buf);
 // then use reader.interface

solved a file read issue for me, but the errors were slightly different.

1 Like
5

Thanks it worked. Also this works:

/// Read all lines from file.
fn readLines(e: *Editor, file: std.fs.File) !void {
    var buf: [1024 * 1024]u8 = undefined;
    var reader = file.reader(&buf);

    while (reader.interface.takeDelimiterExclusive('\n')) |line| {
        try e.insertRow(e.buffer.rows.items.len, line);
    } else |err| if (err != error.EndOfStream) return err;
}
1 Like
6

On save it still doesn’t work. Using this

    const file = std.fs.cwd().createFile(B.filename.?, .{ .truncate = true });
    if (file) |f| {
        var buf: [1024]u8 = undefined;
        var stdout = std.fs.File.stdout().writer(&buf).interface;
        defer f.close();
        stdout.writeAll(contents.items) catch |err| return e.ioerr(err);
        try stdout.flush();
        return;
    }

but getting WriteFailed

Summary 
/home/gianmaria/.local/versions/zig/0.15.1/lib/std/posix.zig:1376:22: 0x11a458e in writev (std.zig)
            .BADF => return error.NotOpenForWriting, // Can be a race condition.
                     ^
/home/gianmaria/.local/versions/zig/0.15.1/lib/std/fs/File.zig:1770:21: 0x119bb8a in drain (std.zig)
                    return error.WriteFailed;
                    ^
/home/gianmaria/.local/versions/zig/0.15.1/lib/std/Io/Writer.zig:316:28: 0x103f7c9 in defaultFlush (std.zig)
    while (w.end != 0) _ = try drainFn(w, &.{""}, 1);
                           ^
/home/gianmaria/.local/versions/zig/0.15.1/lib/std/Io/Writer.zig:310:5: 0x10b35d3 in flush (std.zig)
    return w.vtable.flush(w);
    ^
/home/gianmaria/Progetti/zig/kilo/src/Editor.zig:147:9: 0x11685a6 in saveFile (main.zig)
        try stdout.flush();
        ^
/home/gianmaria/Progetti/zig/kilo/src/Editor.zig:282:20: 0x116a1a9 in processKeypress (main.zig)
        .ctrl_s => try e.saveFile(),
                   ^
/home/gianmaria/Progetti/zig/kilo/src/Editor.zig:73:9: 0x116b089 in startUp (main.zig)
        try e.processKeypress();
        ^
/home/gianmaria/Progetti/zig/kilo/src/main.zig:25:5: 0x116b6c3 in main (main.zig)
    try e.startUp(args.next()); // possible file to open
    ^
7

Ok fixed it like this:

    const file = std.fs.cwd().createFile(B.filename.?, .{ .truncate = true });
    if (file) |f| {
        var buf: [1024]u8 = undefined;
        var writer = f.writer(&buf);
        defer f.close();
        writer.interface.writeAll(contents.items) catch |err| return e.ioerr(err);
        try writer.interface.flush();
        return;
    }
8

I still have a question though… How do I read files with very long lines? If I make the buffer too big it doesn’t fit the stack.

1 Like
9

Try using streamDelimiter with a Writer.Allocating:

const std = @import("std");

pub fn main() !void {
    var alloc = std.heap.DebugAllocator(.{}).init;
    defer _ = alloc.deinit();

    const gpa = alloc.allocator();

    var buf: [1024 * 1024]u8 = undefined;
    var reader = std.fs.File.stdin().reader(&buf);

    var line_writer = std.Io.Writer.Allocating.init(gpa);
    defer line_writer.deinit();

    while (reader.interface.streamDelimiter(&line_writer.writer, '\n')) |_| {
        const line = line_writer.written();
        std.debug.print("{s}\n", .{line});
        line_writer.clearRetainingCapacity(); // empty the line buffer
        reader.interface.toss(1); // skip the newline
    } else |err| if (err != error.EndOfStream) return err;
}
3 Likes
10

Thanks, so this works also with a smaller buffer

/// Read all lines from file.
fn readLines(e: *Editor, file: std.fs.File) !void {
    var buf: [1024]u8 = undefined;
    var reader = file.reader(&buf);

    var line_writer = std.Io.Writer.Allocating.init(e.allocator);
    defer line_writer.deinit();

    while (reader.interface.streamDelimiter(&line_writer.writer, '\n')) |_| {
        try e.insertRow(e.buffer.rows.items.len, line_writer.written());
        line_writer.clearRetainingCapacity(); // empty the line buffer
        reader.interface.toss(1); // skip the newline
    } else |err| if (err != error.EndOfStream) return err;
}

Kind of hard to figure out alone though… Thanks

11 
/// Read all lines from file.
fn readLines(e: *Editor, file: std.fs.File) !void {
   var buf: [1024 * 1024]u8 = undefined;
   var reader = file.reader(&buf).interface;

   while (reader.takeDelimiterExclusive('\n')) |line| {
       try e.insertRow(e.buffer.rows.items.len, line);
   } else |err| if (err != error.EndOfStream) return err;
}

The problem you were running into was that std.fs.File.Reader needs to remain allocated. You made a copy of the std.Io.Reader interface, and then used that. This causes a problem when the std.fs.File.Reader functions do a @fieldParentPtr to find the address of the std.fs.File.Reader from the *std.Io.Reader address.

5 Likes
12

Another option is to allocate the buffer on the heap instead of on the stack:

/// Read all lines from file.
fn readLines(e: *Editor, arena: std.mem.Allocator, file: std.fs.File) !void {
    const buf = try arena.alloc(u8, 64 * 1024 * 1024);
    defer arena.free(buf);
    var reader = file.reader(buf);

    while (reader.interface.takeDelimiterExclusive('\n')) |line| {
        try e.insertRow(e.buffer.rows.items.len, line);
    } else |err| if (err != error.EndOfStream) return err;
}
1 Like
13

Got it. In general it’s really hard to know which intermediates you can use and which you cannot. The error message wasn’t too helpful either. Sometimes you get some const-related errors (understandable) but in this case it was nothing like that.

1 Like
14

Thanks, this works too, pretty much what I needed

/// Read all lines from file.
fn readLines(e: *Editor, file: std.fs.File) !void {
    const buf = try e.allocator.alloc(u8, 60 * 1024 * 1024);
    defer e.allocator.free(buf);
    var reader = file.reader(buf);

    while (reader.interface.takeDelimiterExclusive('\n')) |line| {
        try e.insertRow(e.buffer.rows.items.len, line);
    } else |err| if (err != error.EndOfStream) return err;
}
15

You must not copy the interface or it makes the field parent pointer not work.

16

Just to see if I understand it too:
The copy is happening in the first version of the code here:

var reader = file.reader(&buf).interface;

This mistake is bound to happen quite often, especially if developers are also using OO languages.

Inserting a & in this line after the = would’ve solved it too?

17

lol so my comment did the trick, not the initialization. Sounds like a footgun tbh. Anyways, thanks for the clarification.

18

Yep, but I believe https://github.com/ziglang/zig/issues/2414 will catch most in this class of bugs

1 Like
19

If you take a pointer to a temporary you end up with a const pointer, I don’t think you want a *const Writer, because most of the methods on it require a *Writer so you can actually use it for writing.

1 Like
20

Thanks got it. Maybe hiding dangerous intermediates behind function calls that return underscored members would work in this kind of cases? I just tried replacing in std all usages of interface with _interface, then having methods like:

    // in std.fs.File.Writer
    _interface: std.Io.Writer,

    pub fn interface(w: *Writer) *std.Io.Writer {
        return &w._interface;
    }

    // in std.fs.File.Reader
    _interface: std.Io.Reader,

    pub fn interface(r: *Reader) *std.Io.Reader {
        return &r._interface;
    }

Then this wouldn’t compile:

    var reader = file.reader(buf).interface();

    while (reader.takeDelimiterExclusive('\n')) |line| {

src/Editor.zig|161 col 34| error: expected type '*fs.File.Reader', found '*const fs.File.Reader'

while this would compile and work

    var reader = file.reader(buf);

    while (reader.interface().takeDelimiterExclusive('\n')) |line| {

Just saying… it would be the usual const-related error for which we know the cure.

Or just methods like reader.getInterface(), so that to renaming the field isn’t necessary.

3 Likes