lonjil/derploader - Change 4LTHJQPGOWO7IJ73ATFIRJNFBUIPII55434FSD6OMZBP7J2BOPIQC

Currently dysfunctional on AMD64 Linux due to bus error during `zig build`.

Created by Lonnie on July 31, 2021

4LTHJQPGOWO7IJ73ATFIRJNFBUIPII55434FSD6OMZBP7J2BOPIQC

In channels

main

Change contents

Root addition: 4LTHJQPGOWO7IJ73ATFIRJNFBUIPII55434FSD6OMZBP7J2BOPIQC
File addition: .gitignore (----------)
[0.2]
```
secrets/
*.db3
```

File addition: build.zig (----------)

[0.2]

const std = @import("std");
const deps = @import("./deps.zig");
pub fn build(b: *std.build.Builder) void {
    // Standard target options allows the person running `zig build` to choose
    // what target to build for. Here we do not override the defaults, which
    // means any target is allowed, and the default is native. Other options
    // for restricting supported target set are available.
    var target = b.standardTargetOptions(
        .{
            //.default_target = .{
            //    .abi = .musl,
            //},
        },
    );
    if (target.isGnuLibC()) target.setGnuLibCVersion(2, 28, 0);
    // Standard release options allow the person running `zig build` to select
    // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall.
    const mode = b.standardReleaseOptions();
    const exe = b.addExecutable("derploader", "src/main.zig");
    deps.addAllTo(exe);
    exe.linkLibC();
    exe.linkSystemLibrary("libcurl");
    exe.setTarget(target);
    exe.setBuildMode(mode);
    exe.install();
    const run_cmd = exe.run();
    run_cmd.step.dependOn(b.getInstallStep());
    if (b.args) |args| {
        run_cmd.addArgs(args);
    }
    const run_step = b.step("run", "Run the app");
    run_step.dependOn(&run_cmd.step);
}

File addition: source-me.sh (----------)
[0.2]
```
export PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1
```
File addition: src (d--x------)
[0.2]

File addition: main.zig (----------)

[0.1437]

const std = @import("std");
const sqlite = @import("sqlite");
const clap = @import("clap");
const curl = @cImport({
    @cInclude("curl/curl.h");
});
const log = std.log.scoped(.derploader);
const params = [_]clap.Param(clap.Help){
    clap.parseParam("-h, --help                    Display this help and exit.") catch unreachable,
    clap.parseParam("-c, --create=<PATH>           Create new database at PATH.") catch unreachable,
    clap.parseParam("-m <ID>                       Download metadata for ID image.") catch unreachable,
    clap.parseParam("-d <ID>                       Download image data of ID.") catch unreachable,
};
fn printFullUsage(w: anytype) !void {
    _ = try w.print("{s} ", .{std.os.argv[0]});
    try clap.usage(w, &params);
    _ = try w.writeByte('\n');
    try clap.help(w, &params);
    return;
}
fn sqliteErrorReport(str: []const u8, db: *sqlite.Db) void {
    log.err("{s}: {}", .{ str, db.getDetailedError() });
}
fn curlErrorReport(str: []const u8, code: curl.CURLcode) void {
    log.err("{s}: {s} {s}", .{ str, curl.curl_easy_strerror(code), curlerr[0.. :0] });
}
const create =
    \\CREATE TABLE IF NOT EXISTS image(
    \\  id INTEGER UNIQUE,
    \\  metadata TEXT,
    \\  image BLOB,
    \\  thumb BLOB,
    \\  full_url TEXT GENERATED ALWAYS AS
    \\    (json_extract(metadata, '$.image.representations.full')) VIRTUAL,
    \\  thumb_url TEXT GENERATED ALWAYS AS
    \\    (json_extract(metadata, '$.image.representations.thumb')) VIRTUAL,
    \\  hash_full TEXT,
    \\  hash_thumb TEXT,
    \\  hash_meta TEXT
    \\);
;
const metatable =
    \\CREATE TABLE IF NOT EXISTS derpiloader(
    \\  name TEXT,
    \\  value
    \\);
;
pub fn insertMeta(db: *sqlite.Db, id: u64, meta: []const u8) !void {
    const q =
        \\INSERT OR ROLLBACK INTO image (id, metadata) VALUES (?, ?);
    ;
    try db.exec(q, .{ .id = id, .metadata = meta });
}
const api_base = "https://derpibooru.org/api/v1/json";
var urlbuf = [_:0]u8{0} ** 512;
var curlerr = [_:0]u8{0} ** (curl.CURL_ERROR_SIZE + 1);
const hash_prefix = "blake3-";
var hash_buf = [_]u8{0} ** (std.crypto.hash.Blake3.digest_length);
var hash_buf2 = [_]u8{0} ** (std.crypto.hash.Blake3.digest_length * 2 + hash_prefix[0..].len);
fn hashit(input: []const u8) !void {
    std.crypto.hash.Blake3.hash(input, hash_buf[0..], .{});
    _ = try std.fmt.bufPrint(
        hash_buf2[0..],
        hash_prefix ++ "{s}",
        .{std.fmt.fmtSliceHexLower(hash_buf[0..])},
    );
}
pub fn main() anyerror!void {
    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
    const alloc = &gpa.allocator;
    //const key = try std.process.getEnvVarOwned(alloc, "derpikey");
    var diag = clap.Diagnostic{};
    var args = clap.parse(
        clap.Help,
        &params,
        .{ .diagnostic = &diag, .allocator = alloc },
    ) catch |err| {
        // Report useful error and exit
        diag.report(std.io.getStdErr().writer(), err) catch {};
        return;
    };
    defer args.deinit();
    if (args.flag("-h")) {
        var w = std.io.getStdOut().writer();
        try printFullUsage(w);
        return;
    }
    var db: sqlite.Db = undefined;
    const filename = "test.db3";
    try db.init(.{
        .mode = sqlite.Db.Mode{ .File = filename },
        .open_flags = .{
            .write = true,
            .create = true,
        },
        .threading_mode = .Serialized,
    });
    db.exec(create, .{}) catch sqliteErrorReport("Couldn't create table", &db);
    var ret = curl.curl_global_init(curl.CURL_GLOBAL_ALL);
    if (ret != curl.CURLE_OK) {
        log.err("cURL global init failure: {s}", .{curl.curl_easy_strerror(ret)});
        return;
    }
    defer curl.curl_global_cleanup();
    const handle = curl.curl_easy_init() orelse return error.CURLHandleInitFailed;
    defer curl.curl_easy_cleanup(handle);
    var response_buffer = std.ArrayList(u8).init(alloc);
    defer response_buffer.deinit();
    _ = curl.curl_easy_setopt(handle, curl.CURLOPT_ERRORBUFFER, &curlerr);
    if (args.option("-m")) |id_str| {
        const id = std.fmt.parseInt(u64, id_str, 10) catch {
            log.err("Image ID must be a positive integer.", .{});
            return;
        };
        const foobar = db.one(
            bool,
            "SELECT true FROM image WHERE id = ?",
            .{},
            .{ .id = id },
        ) catch {
            sqliteErrorReport("ID check read error", &db);
            return;
        };
        if (foobar) |_| {
            log.info("Info for id {d} already acquired.", .{id});
            return;
        }
        _ = try std.fmt.bufPrintZ(
            urlbuf[0..],
            api_base ++ "/images/{d}",
            .{id},
        );
        easyFetch(handle, &urlbuf, &response_buffer) catch return;
        //var w = std.io.getStdOut().writer();
        const valid = std.json.validate(response_buffer.items);
        if (valid) {
            try db.exec("BEGIN IMMEDIATE;", .{});
            errdefer db.exec("ROLLBACK;", .{}) catch {};
            insertMeta(&db, id, response_buffer.items) catch {
                sqliteErrorReport("Can't insert:", &db);
                return;
            };
            try hashit(response_buffer.items);
            db.exec(
                "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
                .{ hash_buf2[0..], id },
            ) catch {
                sqliteErrorReport("Couldn't insert", &db);
                return;
            };
            try db.exec("COMMIT", .{});
        }
    }
    if (args.option("-d")) |id_str| {
        const id = std.fmt.parseInt(u64, id_str, 10) catch {
            log.err("Image ID must be a positive integer.", .{});
            return;
        };
        const foobar = db.oneAlloc(
            struct {
                full_url: ?[:0]u8,
                thumb_url: ?[:0]u8,
            },
            alloc,
            "SELECT full_url, thumb_url FROM image WHERE id = ?",
            .{},
            .{ .id = id },
        ) catch {
            sqliteErrorReport("ID check read error", &db);
            return;
        };
        if (foobar) |res| {
            if (res.full_url) |url| {
                easyFetch(handle, url, &response_buffer) catch return;
                try db.exec("BEGIN IMMEDIATE;", .{});
                errdefer db.exec("ROLLBACK;", .{}) catch {};
                db.exec(
                    "UPDATE OR ROLLBACK image SET image = ? WHERE id = ?",
                    .{
                        .image = sqlite.Blob{ .data = response_buffer.items },
                        .id = id,
                    },
                ) catch {
                    sqliteErrorReport("Couldn't add image to DB.", &db);
                    return;
                };
                try hashit(response_buffer.items);
                db.exec(
                    "UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?",
                    .{ hash_buf2[0..], id },
                ) catch {
                    sqliteErrorReport("Couldn't insert", &db);
                    return;
                };
                try db.exec("COMMIT", .{});
            }
        } else {
            log.err("No metadata for id {d} available.", .{id});
            return;
        }
    }
}
fn easyFetch(handle: *curl.CURL, url: [*:0]const u8, resp: *std.ArrayList(u8)) !void {
    var ret = curl.curl_easy_setopt(handle, curl.CURLOPT_URL, url);
    if (ret != curl.CURLE_OK) {
        curlErrorReport("cURL set url:", ret);
        return error.FUCK;
    }
    ret = curl.curl_easy_setopt(handle, curl.CURLOPT_WRITEFUNCTION, writeToArrayListCallback);
    if (ret != curl.CURLE_OK) {
        curlErrorReport("cURL set writefunction:", ret);
        return error.FUCK;
    }
    ret = curl.curl_easy_setopt(handle, curl.CURLOPT_WRITEDATA, resp);
    if (ret != curl.CURLE_OK) {
        curlErrorReport("cURL set writedata:", ret);
        return error.FUCK;
    }
    ret = curl.curl_easy_setopt(handle, curl.CURLOPT_USERAGENT, "Derpiloader 0.1 (linux)");
    if (ret != curl.CURLE_OK) {
        curlErrorReport("cURL set user agent:", ret);
        return error.FUCK;
    }
    ret = curl.curl_easy_perform(handle);
    if (ret != curl.CURLE_OK) {
        curlErrorReport("cURL perform:", ret);
        return error.FUCK;
    }
    log.info("Got {d} bytes", .{resp.items.len});
}
fn writeToArrayListCallback(
    data: *c_void,
    size: c_uint,
    nmemb: c_uint,
    user_data: *c_void,
) callconv(.C) c_uint {
    var buffer = @intToPtr(*std.ArrayList(u8), @ptrToInt(user_data));
    var typed_data = @intToPtr([*]u8, @ptrToInt(data));
    buffer.appendSlice(typed_data[0 .. nmemb * size]) catch return 0;
    return nmemb * size;
}

File addition: main.zig (----------)

[0.1437]

const std = @import("std");
const sqlite = @import("sqlite");
const clap = @import("clap");
const curl = @cImport({
    @cInclude("curl/curl.h");
});
const log = std.log.scoped(.derploader);
const params = [_]clap.Param(clap.Help){
    clap.parseParam("-h, --help                    Display this help and exit.") catch unreachable,
    clap.parseParam("-c, --create=<PATH>           Create new database at PATH.") catch unreachable,
    clap.parseParam("-m <ID>                       Download metadata for ID image.") catch unreachable,
    clap.parseParam("-d <ID>                       Download image data of ID.") catch unreachable,
};
fn printFullUsage(w: anytype) !void {
    _ = try w.print("{s} ", .{std.os.argv[0]});
    try clap.usage(w, &params);
    _ = try w.writeByte('\n');
    try clap.help(w, &params);
    return;
}
fn sqliteErrorReport(str: []const u8, db: *sqlite.Db) void {
    log.err("{s}: {}", .{ str, db.getDetailedError() });
}
fn curlErrorReport(str: []const u8, code: curl.CURLcode) void {
    log.err("{s}: {s} {s}", .{ str, curl.curl_easy_strerror(code), curlerr[0.. :0] });
}
const create =
    \\CREATE TABLE IF NOT EXISTS image(
    \\  id INTEGER UNIQUE,
    \\  metadata TEXT,
    \\  image BLOB,
    \\  thumb BLOB,
    \\  full_url TEXT GENERATED ALWAYS AS
    \\    (json_extract(metadata, '$.image.representations.full')) VIRTUAL,
    \\  thumb_url TEXT GENERATED ALWAYS AS
    \\    (json_extract(metadata, '$.image.representations.thumb')) VIRTUAL,
    \\  hash_full TEXT,
    \\  hash_thumb TEXT,
    \\  hash_meta TEXT
    \\);
;
const metatable =
    \\CREATE TABLE IF NOT EXISTS derpiloader(
    \\  name TEXT,
    \\  value
    \\);
;
pub fn insertMeta(db: *sqlite.Db, id: u64, meta: []const u8) !void {
    const q =
        \\INSERT OR ROLLBACK INTO image (id, metadata) VALUES (?, ?);
    ;
    try db.exec(q, .{ .id = id, .metadata = meta });
}
const api_base = "https://derpibooru.org/api/v1/json";
var urlbuf = [_:0]u8{0} ** 512;
var curlerr = [_:0]u8{0} ** (curl.CURL_ERROR_SIZE + 1);
const hash_prefix = "blake3-";
var hash_buf = [_]u8{0} ** (std.crypto.hash.Blake3.digest_length);
var hash_buf2 = [_]u8{0} ** (std.crypto.hash.Blake3.digest_length * 2 + hash_prefix[0..].len);
fn hashit(input: []const u8) !void {
    std.crypto.hash.Blake3.hash(input, hash_buf[0..], .{});
    _ = try std.fmt.bufPrint(
        hash_buf2[0..],
        hash_prefix ++ "{s}",
        .{std.fmt.fmtSliceHexLower(hash_buf[0..])},
    );
}
pub fn main() anyerror!void {
    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
    const alloc = &gpa.allocator;
    //const key = try std.process.getEnvVarOwned(alloc, "derpikey");
    var diag = clap.Diagnostic{};
    var args = clap.parse(
        clap.Help,
        &params,
        .{ .diagnostic = &diag, .allocator = alloc },
    ) catch |err| {
        // Report useful error and exit
        diag.report(std.io.getStdErr().writer(), err) catch {};
        return;
    };
    defer args.deinit();
    if (args.flag("-h")) {
        var w = std.io.getStdOut().writer();
        try printFullUsage(w);
        return;
    }
    var db: sqlite.Db = undefined;
    const filename = "test.db3";
    try db.init(.{
        .mode = sqlite.Db.Mode{ .File = filename },
        .open_flags = .{
            .write = true,
            .create = true,
        },
        .threading_mode = .Serialized,
    });
    db.exec(create, .{}) catch sqliteErrorReport("Couldn't create table", &db);
    var ret = curl.curl_global_init(curl.CURL_GLOBAL_ALL);
    if (ret != curl.CURLE_OK) {
        log.err("cURL global init failure: {s}", .{curl.curl_easy_strerror(ret)});
        return;
    }
    defer curl.curl_global_cleanup();
    const handle = curl.curl_easy_init() orelse return error.CURLHandleInitFailed;
    defer curl.curl_easy_cleanup(handle);
    var response_buffer = std.ArrayList(u8).init(alloc);
    defer response_buffer.deinit();
    _ = curl.curl_easy_setopt(handle, curl.CURLOPT_ERRORBUFFER, &curlerr);
    if (args.option("-m")) |id_str| {
        const id = std.fmt.parseInt(u64, id_str, 10) catch {
            log.err("Image ID must be a positive integer.", .{});
            return;
        };
        const foobar = db.one(
            bool,
            "SELECT true FROM image WHERE id = ?",
            .{},
            .{ .id = id },
        ) catch {
            sqliteErrorReport("ID check read error", &db);
            return;
        };
        if (foobar) |_| {
            log.info("Info for id {d} already acquired.", .{id});
            return;
        }
        _ = try std.fmt.bufPrintZ(
            urlbuf[0..],
            api_base ++ "/images/{d}",
            .{id},
        );
        easyFetch(handle, &urlbuf, &response_buffer) catch return;
        //var w = std.io.getStdOut().writer();
        const valid = std.json.validate(response_buffer.items);
        if (valid) {
            try db.exec("BEGIN IMMEDIATE;", .{});
            errdefer db.exec("ROLLBACK;", .{}) catch {};
            insertMeta(&db, id, response_buffer.items) catch {
                sqliteErrorReport("Can't insert:", &db);
                return;
            };
            try hashit(response_buffer.items);
            db.exec(
                "UPDATE OR ROLLBACK image SET hash_meta = ? WHERE id = ?",
                .{ hash_buf2[0..], id },
            ) catch {
                sqliteErrorReport("Couldn't insert", &db);
                return;
            };
            try db.exec("COMMIT", .{});
        }
    }
    if (args.option("-d")) |id_str| {
        const id = std.fmt.parseInt(u64, id_str, 10) catch {
            log.err("Image ID must be a positive integer.", .{});
            return;
        };
        const foobar = db.oneAlloc(
            struct {
                full_url: ?[:0]u8,
                thumb_url: ?[:0]u8,
            },
            alloc,
            "SELECT full_url, thumb_url FROM image WHERE id = ?",
            .{},
            .{ .id = id },
        ) catch {
            sqliteErrorReport("ID check read error", &db);
            return;
        };
        if (foobar) |res| {
            if (res.full_url) |url| {
                easyFetch(handle, url, &response_buffer) catch return;
                try db.exec("BEGIN IMMEDIATE;", .{});
                errdefer db.exec("ROLLBACK;", .{}) catch {};
                db.exec(
                    "UPDATE OR ROLLBACK image SET image = ? WHERE id = ?",
                    .{
                        .image = sqlite.Blob{ .data = response_buffer.items },
                        .id = id,
                    },
                ) catch {
                    sqliteErrorReport("Couldn't add image to DB.", &db);
                    return;
                };
                try hashit(response_buffer.items);
                db.exec(
                    "UPDATE OR ROLLBACK image SET hash_full = ? WHERE id = ?",
                    .{ hash_buf2[0..], id },
                ) catch {
                    sqliteErrorReport("Couldn't insert", &db);
                    return;
                };
                try db.exec("COMMIT", .{});
            }
        } else {
            log.err("No metadata for id {d} available.", .{id});
            return;
        }
    }
}
fn easyFetch(handle: *curl.CURL, url: [*:0]const u8, resp: *std.ArrayList(u8)) !void {
    var ret = curl.curl_easy_setopt(handle, curl.CURLOPT_URL, url);
    if (ret != curl.CURLE_OK) {
        curlErrorReport("cURL set url:", ret);
        return error.FUCK;
    }
    ret = curl.curl_easy_setopt(handle, curl.CURLOPT_WRITEFUNCTION, writeToArrayListCallback);
    if (ret != curl.CURLE_OK) {
        curlErrorReport("cURL set writefunction:", ret);
        return error.FUCK;
    }
    ret = curl.curl_easy_setopt(handle, curl.CURLOPT_WRITEDATA, resp);
    if (ret != curl.CURLE_OK) {
        curlErrorReport("cURL set writedata:", ret);
        return error.FUCK;
    }
    ret = curl.curl_easy_setopt(handle, curl.CURLOPT_USERAGENT, "Derpiloader 0.1 (linux)");
    if (ret != curl.CURLE_OK) {
        curlErrorReport("cURL set user agent:", ret);
        return error.FUCK;
    }
    ret = curl.curl_easy_perform(handle);
    if (ret != curl.CURLE_OK) {
        curlErrorReport("cURL perform:", ret);
        return error.FUCK;
    }
    log.info("Got {d} bytes", .{resp.items.len});
}
fn writeToArrayListCallback(
    data: *c_void,
    size: c_uint,
    nmemb: c_uint,
    user_data: *c_void,
) callconv(.C) c_uint {
    var buffer = @intToPtr(*std.ArrayList(u8), @ptrToInt(user_data));
    var typed_data = @intToPtr([*]u8, @ptrToInt(data));
    buffer.appendSlice(typed_data[0 .. nmemb * size]) catch return 0;
    return nmemb * size;
}

File addition: zig.mod (----------)

[0.2]

id: 1m1xcon88qpj9vi17d8yo2743g0pqswk91ep21b0v139bu5s
name: derploader
main: src/main.zig
dev_dependencies:
    - src: git https://github.com/Hejsil/zig-clap branch-zig-master
    - src: git https://github.com/vrischmann/zig-sqlite
      c_source_flags:
          - -DSQLITE_ENABLE_JSON1

File addition: zigmod.lock (----------)

[0.2]

2
git https://github.com/Hejsil/zig-clap branch-zig-master
git https://github.com/vrischmann/zig-sqlite commit-fafe666f22590faf82d7c7ce1087858002c004b6

File addition: zigmod.sum (----------)

[0.2]

blake3-e59408623c62ac2b958cf33f848cb9dccc87631e8ca98e479cf9870694a4771d v/git/github.com/Hejsil/zig-clap/branch-zig-master
blake3-605156fee0fdf3be71878c38e7bb1e1934ba4bafeaf5fde535685bfa924345ea git/github.com/vrischmann/zig-sqlite

Currently dysfunctional on AMD64 Linux due to bus error during `zig build`.

In channels

Change contents

Root addition: 4LTHJQPGOWO7IJ73ATFIRJNFBUIPII55434FSD6OMZBP7J2BOPIQC

File addition: .gitignore (----------)

File addition: build.zig (----------)

File addition: source-me.sh (----------)

File addition: src (d--x------)

File addition: main.zig (----------)

File addition: main.zig (----------)

File addition: zig.mod (----------)

File addition: zigmod.lock (----------)

File addition: zigmod.sum (----------)