I am probably overthinking something, but I’ve hit a bit of a quandary while trying to support peek for a base64 Io.Reader implementation. The issue is that peek expects stream to fill up to the full buffer for the Io.Reader. But the base64 decoding ratio is 4 to 3, which can result in needing to read more than one byte from the input reader to fill the next byte in the buffer. So on a peek that fills up the buffer with a partial chunk, the full chunk needs to stay in a buffer somewhere for the next time stream reads from the input reader. Further, there is no guarantee that the upstream reader’s buffer is large enough to hold a chunk, and there is no way to use the interface’s buffer since the entire buffer can be used by the interface.
The solution I have come up with is to wrap the input reader in an indirect reader that has a chunk length buffer:
pub const Reader = struct {
pub const min_buffer_len = 3;
pub const chunk_len = 4;
in: *Io.Reader,
decoder: base64.Base64Decoder,
interface: Io.Reader,
err: ?base64.Error,
indirect: ?Indirect,
decoded_offset: u2,
const Indirect = ReaderIndirect(chunk_len);
pub fn init(in: *Io.Reader, decoder: base64.Base64Decoder, buffer: []u8) Reader {
assert(buffer.len >= min_buffer_len);
return .{
.in = in,
.decoder = decoder,
.interface = .{
.vtable = &.{
.stream = stream,
.readVec = readVec,
},
.buffer = buffer,
.end = 0,
.seek = 0,
},
.err = null,
.indirect = null,
.decoded_offset = 0,
};
}
fn stream(r: *Io.Reader, _: *Io.Writer, _: Io.Limit) Io.Reader.StreamError!usize {
try readChunk(r);
return 0;
}
fn readVec(r: *Io.Reader, _: [][]u8) Io.Reader.Error!usize {
try readChunk(r);
return 0;
}
fn readChunk(r: *Io.Reader) Io.Reader.Error!void {
const self: *Reader = @fieldParentPtr("interface", r);
var in = if (self.in.buffer.len >= chunk_len) self.in else in: {
if (self.indirect == null) {
Indirect.init(&self.indirect, self.in);
}
break :in &self.indirect.?.interface;
};
// peek a chunk, or at least one byte worth of decoded data (2 bytes).
const encoded = enc: {
const buf = try in.peekGreedy(2);
break :enc buf[0..@min(buf.len, chunk_len)];
};
const eos_seen = (encoded.len < chunk_len);
assert(encoded.len <= chunk_len);
const decoded_len = self.decoder.calcSizeForSlice(encoded) catch |err| {
if (eos_seen) return error.EndOfStream;
self.err = err;
return error.ReadFailed;
};
assert(decoded_len <= Writer.chunk_len);
if (eos_seen and decoded_len == 0) return error.EndOfStream;
try r.rebase(1);
const decoded = dec: {
var buf: [chunk_len]u8 = undefined;
const chunk = buf[0..decoded_len];
self.decoder.decode(chunk, encoded) catch |err| {
if (eos_seen) return error.EndOfStream;
self.err = err;
return error.ReadFailed;
};
break :dec chunk;
};
var offset: usize = @intCast(self.decoded_offset);
assert(offset < decoded_len);
const remaining = decoded_len - offset;
const n = @min(r.buffer.len - r.end, remaining);
@memcpy(r.buffer[r.end..][0..n], decoded[offset..][0..n]);
r.end += n;
offset += n;
const chunk_delivered = offset == decoded_len;
if (chunk_delivered) {
in.toss(encoded.len);
self.decoded_offset = 0;
} else {
self.decoded_offset = @intCast(offset);
}
}
};
But I’m really unhappy with the ergonomics of ReaderIndirect, which can’t be safely copied and so it can’t initialize it’s (self-referential) interface field until it can access *Self:
fn ReaderIndirect(buffer_len: comptime_int) type {
return struct {
const Self = @This();
in: *Io.Reader,
interface: Io.Reader,
buffer: [buffer_len]u8,
pub fn init(self: *?Self, in: *Io.Reader) void {
self.* = .{
.in = in,
.interface = undefined,
.buffer = undefined,
};
self.*.?.interface = .{
.vtable = &.{
.stream = Self.stream,
.readVec = Self.readVec,
},
.buffer = &self.*.?.buffer,
.seek = 0,
.end = 0,
};
}
// the rest is copied from testing.ReaderIndirect
fn readVec(r: *Io.Reader, _: [][]u8) Io.Reader.Error!usize {
try streamInner(r);
return 0;
}
fn stream(r: *Io.Reader, _: *Io.Writer, _: Io.Limit) Io.Reader.StreamError!usize {
try streamInner(r);
return 0;
}
fn streamInner(r: *Io.Reader) Io.Reader.Error!void {
const r_indirect: *@This() = @alignCast(@fieldParentPtr("interface", r));
// If there's no room remaining in the buffer at all, make room.
if (r.buffer.len == r.end) {
try r.rebase(r.buffer.len);
}
var writer: Io.Writer = .{
.buffer = r.buffer,
.end = r.end,
.vtable = &.{
.drain = Io.Writer.unreachableDrain,
.rebase = Io.Writer.unreachableRebase,
},
};
defer r.end = writer.end;
r_indirect.in.streamExact(&writer, r.buffer.len - r.end) catch |err| switch (err) {
// Only forward EndOfStream if no new bytes were written to the buffer
error.EndOfStream => |e| if (r.end == writer.end) {
return e;
},
error.WriteFailed => unreachable,
else => |e| return e,
};
}
};
}
It seems like any reader implementation that needs to read past it’s buffers
length of bytes from it’s input reader has this problem, are there other
simpler solutions to this out there?