How to make an HTTP GET request?

Help
1

Hello, I’m trying to make a simple HTTP GET request with std.http however the following keeps failing with an OutOfMemory error using a FixedBufferAllocator. If I swap it out for std.heap.c_allocator it works. Can anybody point me to the right direction? My reference for this code was this answer. I tried increasing the size to 100kb but that also didn’t change anything.

web_req.zig

const std = @import("std");
const log = std.log;
const uri = std.Uri;
const http = std.http;

pub fn main() !void {
    // 8kb in stack
    var buff: [8 * 1024]u8 = undefined;
    var fba = std.heap.FixedBufferAllocator.init(&buff);
    var tsa = std.heap.ThreadSafeAllocator{ .child_allocator = fba.allocator() };
    const a = tsa.allocator();

    log.info("Parsing url\n", .{});
    const url = uri.parse("https://raw.githubusercontent.com/keroserene/rickrollrc/master/roll.sh") catch |err| {
        log.err("Parse error {!}\n", .{err});
        return err;
    };

    log.info("Creating client", .{});
    var client = http.Client{ .allocator = a };
    defer client.deinit();

    log.info("Creating server buffer", .{});
    var sh_buff: [4 * 1024]u8 = undefined;

    log.info("Creating request", .{});
    var req = try client.open(.GET, url, .{ .server_header_buffer = &sh_buff });
    defer req.deinit();

    log.info("Sending request", .{});
    try req.send();

    log.info("Finishing request", .{});
    try req.finish();

    log.info("Waiting for request", .{});
    try req.wait();

    log.info("Reading response", .{});
    const response = try req.reader().readAllAlloc(a, 1024 * 7);
    defer a.free(response);
    const resp_parsed = uri.percentDecodeInPlace(response);

    const stdout = std.io.getStdOut().writer();
    try stdout.print("Response:\n{s}", .{resp_parsed});
}

And output

cocoa@TheTreehouse ~/w/P/z/wheelmaker [1]> zig run -lc web_req.zig                                                                (self)
info: Parsing url

info: Creating client
info: Creating server buffer
info: Creating request
error: CertificateBundleLoadFailure
/home/cocoa/.local/bin/zig-linux-x86_64-0.13.0/lib/std/mem/Allocator.zig:225:89: 0x1230f26 in allocBytesWithAlignment__anon_10509 (web_req)
    const byte_ptr = self.rawAlloc(byte_count, log2a(alignment), return_address) orelse return Error.OutOfMemory;
                                                                                        ^
/home/cocoa/.local/bin/zig-linux-x86_64-0.13.0/lib/std/mem/Allocator.zig:211:5: 0x116264a in allocWithSizeAndAlignment__anon_9829 (web_req)
    return self.allocBytesWithAlignment(alignment, byte_count, return_address);
    ^
/home/cocoa/.local/bin/zig-linux-x86_64-0.13.0/lib/std/mem/Allocator.zig:193:5: 0x115f17a in alignedAlloc__anon_9343 (web_req)
    return self.allocAdvancedWithRetAddr(T, alignment, n, @returnAddress());
    ^
/home/cocoa/.local/bin/zig-linux-x86_64-0.13.0/lib/std/array_list.zig:1081:36: 0x138ed96 in ensureTotalCapacityPrecise (web_req)
                const new_memory = try allocator.alignedAlloc(T, alignment, new_capacity);
                                   ^
/home/cocoa/.local/bin/zig-linux-x86_64-0.13.0/lib/std/array_list.zig:1058:13: 0x1314932 in ensureTotalCapacity (web_req)
            return self.ensureTotalCapacityPrecise(allocator, better_capacity);
            ^
/home/cocoa/.local/bin/zig-linux-x86_64-0.13.0/lib/std/array_list.zig:1096:13: 0x12a812e in ensureUnusedCapacity (web_req)
            return self.ensureTotalCapacity(allocator, try addOrOom(self.items.len, additional_count));
            ^
/home/cocoa/.local/bin/zig-linux-x86_64-0.13.0/lib/std/crypto/Certificate/Bundle.zig:233:5: 0x1239848 in addCertsFromFile (web_req)
    try cb.bytes.ensureUnusedCapacity(gpa, needed_capacity);
    ^
/home/cocoa/.local/bin/zig-linux-x86_64-0.13.0/lib/std/crypto/Certificate/Bundle.zig:203:5: 0x1170e81 in addCertsFromFilePathAbsolute (web_req)
    return addCertsFromFile(cb, gpa, file);
    ^
/home/cocoa/.local/bin/zig-linux-x86_64-0.13.0/lib/std/crypto/Certificate/Bundle.zig:104:29: 0x11530cf in rescanLinux (web_req)
                else => |e| return e,
                            ^
/home/cocoa/.local/bin/zig-linux-x86_64-0.13.0/lib/std/crypto/Certificate/Bundle.zig:61:19: 0x1116b21 in rescan (web_req)
        .linux => return rescanLinux(cb, gpa),
                  ^
/home/cocoa/.local/bin/zig-linux-x86_64-0.13.0/lib/std/http/Client.zig:1634:17: 0x110cc9a in open (web_req)
                return error.CertificateBundleLoadFailure;
                ^
/home/cocoa/workspace/Projects/zig/wheelmaker/web_req.zig:27:15: 0x110bafe in main (web_req)
    var req = try client.open(.GET, url, .{ .server_header_buffer = &sh_buff });
              ^
1 Like
2

try using a different allocator like GeneralPurposeAllocator

2 Likes
3

Yes that does work, but I think it’s supposed to be deprecated in 0.14 no? And Choosing an allocator suggests using that as a last option in their checklist. I figured a fixed buffer of 8kb should be enough? I’m just downloading a text file. And I tried increasing it to 100kb as well, but that didn’t solve the issue.

4

It seems like you’ve answered your own question. http.Client needs to be able to allocate more than 8KiB (or 100KiB).

You could find the exact amount needed using the counting aspect of std.testing.FailingAllocator like so:

    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
    defer std.debug.assert(gpa.deinit() == .ok);
    var counting_allocator = std.testing.FailingAllocator.init(gpa.allocator(), .{});
    const a = counting_allocator.allocator();

If I print out counting_allocator.allocated_bytes at the end I get 199400 bytes (194.7KiB). However, I believe the amount required might vary from system to system, since it’s loading certs from your system.

1 Like
5

i was assuming you were on 0.13 as its stable

depends on what 0.14 version you have, it might still be GeneralPurposeAllocator if your version is old enough, or it will have been renamed to DebugAllocator since it’s not the only general one anymore

1 Like
6

GPA won’t be deprecated until after 0.14. in 0.14 there is an alias to the DebugAllocator

7

You should also know that free on the fixed buffer allocator does nothing. So the true memory required when using fixed buffer allocator can sometimes be much more than with an allocator that can truly reclaim memory on free.

1 Like
8

The fixed buffer allocator (and arena allocator) actually do free memory if you give it the last allocation, not that useful in many cases but it does do something.

1 Like