change: Implement decoding of edits, atoms, positions, among others

laumann

Oct 31, 2022, 12:18 PM

ZKAOPMCHBGP7J7NQB233AQJQDX6TBD7REPLF3KN3EIKZK6TZZMBQC

Dependencies

• [2] 2U7P5SFQ Change struct names "struct foo -> typedef struct Foo"
• [3] Y26WT3ZF change: decode message, description and timestamp
• [4] YG4DZB3A add representation of hash, decode dependencies
• [5] OBKF6SII change: decompress the hashed section too
• [6] DKX3RO3L base32: Add encoding of hashes, print base32-encoded hashes
• [7] QYRJIOYP change: separate decoding and printing of hashed struct
• [8] VKLGQREY change: add base32 decode, initial deconstruction of hashed
• [9] RIWSVVAS change: decompress the 'contents' with zstd_seekable
• [10] IS5A7VT6 change: print hashes
• [11] JAGXXSR7 change: initial work on decoding hunks
• [12] Q7TKZCJP Add initial support for reading the offsets from a (fixed) change
• [13] WMFNGOYT change: reduce printed noise, rework some code
• [14] LVX6U4EK change: decode metadata bytes
• [15] WABDGHPO hash: fix unused variable warning
• [16] FMYCPGKD change: extract hash list decoding into reused function
• [17] XTKRT6OQ format the codebase
• [18] 4RYULBDD bincode: introduce a bincode_state struct

Change contents

• replacement in vertex.h at line 7
  [3.124]→[3.124:158](∅→∅)
  − typedef uint64_t change_position;
  [3.124]

  [3.158]
  + typedef uint64_t changepos;
• edit in vertex.h at line 21
  [2.17]

  [2.17]
  + Hash change;

  + changepos pos;
• edit in vertex.h at line 24
  [2.29]

  [3.416]
  +

  + typedef struct {

  + size_t len;

  + Position first; /* Reserve space for the first Position, most changes appear to have just one */

  + Position *entries;

  + } Positionlist;
• replacement in vertex.h at line 36
  [2.47]→[3.555:621](∅→∅),[3.555]→[3.555:621](∅→∅)
  − void *change;

  − change_position start, end; /* line identifier */
  [2.47]

  [2.48]
  + Hash change;

  + changepos start, end; /* line identifier */
• edit in hunk.h at line 6
  [2.109]

  [2.109]
  + unsigned int line; /* This is defined as usize, but it's probably better as a u64 */
• replacement in hunk.h at line 30
  [2.210]→[3.1015:1031](∅→∅),[3.1015]→[3.1015:1031](∅→∅)
  − uint64_t atom;
  [2.210]

  [2.211]
  + Atom change;

  + Local local;

  + str encoding;
• replacement in hunk.h at line 47
  [2.371]→[3.1263:1272](∅→∅),[3.1263]→[3.1263:1272](∅→∅)
  − } hunk;
  [2.371]

  [2.372]
  + };
• edit in hunk.h at line 50
  [3.1276]

  [2.385]
  + typedef struct {

  + size_t len;

  + Basehunk *entries;

  + } Hunklist;

  +
• edit in hunk.h at line 56
  [2.435]

  [3.1332]
  + void hunk_hunklistinit(Hunklist *, size_t);
• edit in hunk.c at line 5
  [2.457]

  [3.1413]
  + #include "scaffold.h"

  + #include "hash.h"

  + #include "vertex.h"

  + #include "atom.h"
• edit in hunk.c at line 44
  [3.2149]

  [3.2149]
  + }

  +

  + void

  + hunk_hunklistinit(Hunklist *hunklist, size_t len)

  + {

  + hunklist->len = len;

  + hunklist->entries = xmalloc(sizeof(Basehunk) * len);
• edit in hash.h at line 19
  [3.288]

  [2.580]
  + void hashprintln(Hash *);
• edit in hash.c at line 11
  [3.445]

  [2.626]
  + hashprintln(Hash *hash)

  + {

  + hashprint(hash);

  + printf("\n");

  + }

  +

  + void
• replacement in hash.c at line 29
  [2.692]→[3.132:159](∅→∅),[3.132]→[3.132:159](∅→∅)
  − printf("%s\n", b32hash);
  [2.692]

  [3.666]
  + printf("%s", b32hash);
• edit in change.h at line 73
  [2.2374]

  [2.2374]
  + uint8_t *metadata;

  + Hunklist hunks;
• edit in change.c at line 15
  [3.836]

  [3.1192]
  + #include "vertex.h"

  + #include "atom.h"

  + #include "hunk.h"
• edit in change.c at line 21
  [3.2152]→[3.2152:2208](∅→∅)
  − #include "atom.h"

  − #include "vertex.h"

  − #include "hunk.h"
• replacement in change.c at line 22
  [3.1]→[3.1:6](∅→∅)
  − void
  [3.1]

  [3.6]
  + static void
• replacement in change.c at line 155
  [3.1]→[3.1:6](∅→∅)
  − void
  [3.1]

  [2.2510]
  + static void
• replacement in change.c at line 169
  [3.371]→[3.371:514](∅→∅)
  − memcpy(target->entries[i].bytes, binstat->buf,

  − BLAKE3_BYTES);

  − binstat->buf += BLAKE3_BYTES;

  − binstat->avail -= BLAKE3_BYTES;
  [3.371]

  [3.514]
  + bincode_getbytes(

  + binstat, target->entries[i].bytes, BLAKE3_BYTES

  + );
• replacement in change.c at line 182
  [3.2252]→[3.2252:2257](∅→∅),[3.2257]→[2.2619:2660](∅→∅)
  − void

  − change_decode_position(Bincode *binstat)
  [3.2252]

  [3.2311]
  + static void

  + change_decode_position(Bincode *binstat, Position *pos)
• edit in change.c at line 185
  [3.2313]→[2.2661:2722](∅→∅),[2.2722]→[3.2380:2381](∅→∅),[3.2380]→[3.2380:2381](∅→∅)
  − uint64_t changepos; /* FIXME this should be its own type */

  −
• replacement in change.c at line 186
  [3.2412]→[2.2723:2736](∅→∅),[2.2736]→[3.2432:2509](∅→∅),[3.2432]→[3.2432:2509](∅→∅)
  − Hash hash;

  − hash.variant = (uint8_t)bincode_getu32(binstat);

  − switch (hash.variant) {
  [3.2412]

  [3.2509]
  + pos->change.variant = (uint8_t)bincode_getu32(binstat);

  + switch (pos->change.variant) {

  + case HASH_BLAKE3:

  + bincode_getbytes(

  + binstat, (uint8_t *)&pos->change.bytes,

  + BLAKE3_BYTES

  + );

  + break;
• edit in change.c at line 196
  [3.2537]

  [3.2537]
  + default:

  + abort();

  + }

  + }

  + pos->pos = bincode_getu64(binstat);

  + }

  +

  + static void

  + change_decode_vertex(Bincode *binstat, Vertex *v)

  + {

  + Hash *hash;

  +

  + hash = &v->change;

  +

  + uint8_t ishash = bincode_getu8(binstat);

  + if (ishash) {

  + hash->variant = (uint8_t)bincode_getu32(binstat);

  + switch (hash->variant) {
• replacement in change.c at line 216
  [3.2578]→[3.2578:2627](∅→∅)
  − binstat, (uint8_t *)hash.bytes, BLAKE3_BYTES
  [3.2578]

  [3.2627]
  + binstat, (uint8_t *)hash->bytes, BLAKE3_BYTES
• edit in change.c at line 219
  [3.2643]

  [3.2643]
  + case HASH_NONE:

  + break;
• replacement in change.c at line 224
  [3.2670]→[2.2737:2757](∅→∅)
  − hashprint(&hash);
  [3.2670]

  [3.2691]
  + }

  + v->start = bincode_getu64(binstat);

  + v->end = bincode_getu64(binstat);

  + if (ishash) {

  + printf("V(Some(");

  + hashprint(hash);

  + printf(")[%lu:%lu])\n", v->start, v->end);

  + } else {

  + printf("V(None)\n");
• edit in change.c at line 234
  [3.2694]→[2.2758:2841](∅→∅)
  − changepos = bincode_getu64(binstat);

  − printf("ChangePosition(%lu)\n", changepos);
• replacement in change.c at line 236
  [3.2792]→[3.2792:2797](∅→∅),[3.2797]→[2.2842:2882](∅→∅)
  − void

  − change_decode_newedge(Bincode *binstat)
  [3.2792]

  [3.2850]
  + static void

  + change_decode_introducedby(Bincode *binstat, Hash *hash)
• replacement in change.c at line 239
  [3.2852]→[3.2852:2877](∅→∅)
  − uint8_t previous, flag;
  [3.2852]

  [3.2877]
  + uint8_t ishash;
• replacement in change.c at line 241
  [3.2878]→[3.2878:3053](∅→∅)
  − dump_buf("newedge", binstat->buf, binstat->avail);

  − previous = bincode_getu8(binstat);

  − flag = bincode_getu8(binstat);

  − printf("previous = %u, flag = %u\n", previous, flag);
  [3.2878]

  [3.3053]
  + ishash = bincode_getu8(binstat);

  + if (ishash) {

  + hash->variant = (uint8_t)bincode_getu32(binstat);

  + switch (hash->variant) {

  + case HASH_BLAKE3:

  + bincode_getbytes(

  + binstat, (uint8_t *)hash->bytes, BLAKE3_BYTES

  + );

  + break;

  + case HASH_NONE:

  + break;

  + default:

  + abort();

  + }

  + printf("introduced_by: ");

  + if (ishash) {

  + printf("Some(");

  + hashprint(hash);

  + printf(")\n");

  + } else {

  + printf("None\n");

  + }

  + }

  + }

  + static void

  + change_decode_newedge(Bincode *binstat, Edge *edge)

  + {

  + edge->previous = bincode_getu8(binstat);

  + edge->flag = bincode_getu8(binstat);

  + printf("previous = %u, flag = %u\n", edge->previous, edge->flag);
• replacement in change.c at line 272
  [3.3113]→[3.3113:3192](∅→∅)
  − /* from: Position<Change = Option<Hash>> */

  − change_decode_position(binstat);
  [3.3113]

  [3.3192]
  +

  + change_decode_position(binstat, &edge->from);

  +

  + change_decode_vertex(binstat, &edge->to);

  +

  + change_decode_introducedby(binstat, &edge->introducedby);

  + }

  +

  + static void

  + change_decode_positionlist(Bincode *binstat, Positionlist *poslist)

  + {

  + size_t len;

  + int i;
• replacement in change.c at line 286
  [3.3193]→[3.3193:3261](∅→∅)
  − /* to: V(Some(HASH)[range?]) */

  − /* introduced_by: Option<Hash> */
  [3.3193]

  [3.3261]
  + poslist->len = len = bincode_getu64(binstat);

  + if (len > 1)

  + poslist->entries = xmalloc(sizeof(Position) * (len - 1));

  + if (len > 0) {

  + change_decode_position(binstat, &poslist->first);

  + for (i = 0; i < len - 1; i++)

  + change_decode_position(binstat, &poslist->entries[i]);

  + }
• replacement in change.c at line 296
  [3.3264]→[3.3264:3269](∅→∅),[3.3269]→[2.2883:2920](∅→∅)
  − void

  − change_decode_atom(Bincode *binstat)
  [3.3264]

  [3.3319]
  + static void

  + change_decode_newvertex(Bincode *binstat, Newvertex *newvertex)
• replacement in change.c at line 299
  [3.3321]→[3.3321:3342](∅→∅)
  − uint64_t edges_len;
  [3.3321]

  [3.3342]
  + /* Fields: upcontext: Vec<Position>, downcontext: Vec<Position>, flag, start, end: ChangePosition, inode: Position */

  +

  + printf("Decoding Newvertex\n");

  + change_decode_positionlist(binstat, &newvertex->upcontext);

  + change_decode_positionlist(binstat, &newvertex->downcontext);

  +

  + newvertex->flag = bincode_getu8(binstat);

  + newvertex->start = bincode_getu64(binstat);

  + newvertex->end = bincode_getu64(binstat);

  +

  + change_decode_position(binstat, &newvertex->inode);

  + }

  +

  + static void

  + change_decode_atom(Bincode *binstat, Atom *atom)

  + {
• replacement in change.c at line 318
  [3.3387]→[3.3387:3439](∅→∅)
  − case 0: /* NewVertex<Option<Hash>> */

  − /* TODO */
  [3.3387]

  [3.3439]
  + case 0:

  + atom->atomtype = NEW_VERTEX;

  + change_decode_newvertex(binstat, &atom->newvertex);
• replacement in change.c at line 324
  [3.3481]→[3.3481:3674](∅→∅)
  − edges_len = bincode_getu64(binstat);

  − printf("EdgeMap (%lu elements)\n", edges_len);

  − for (i = 0; i < edges_len; i++) {

  − change_decode_newedge(binstat);

  − break; /* FIXME remove this */
  [3.3481]

  [3.3674]
  + atom->atomtype = EDGE_MAP;

  + atom->edgemap.edges.len = bincode_getu64(binstat);

  + atom->edgemap.edges.entries =

  + xmalloc(sizeof(Edge) * atom->edgemap.edges.len);

  + printf("EdgeMap (%lu elements)\n", atom->edgemap.edges.len);

  + for (i = 0; i < atom->edgemap.edges.len; i++) {

  + change_decode_newedge(

  + binstat, &atom->edgemap.edges.entries[i]

  + );
• edit in change.c at line 334
  [3.3678]

  [3.3678]
  + /* inode */

  + change_decode_position(binstat, &atom->edgemap.inode);
• edit in change.c at line 337
  [3.3679]→[3.3679:3721](∅→∅)
  − /* A Position<Change = Option<Hash>> */
• edit in change.c at line 340
  [3.3751]

  [3.3751]
  + }

  + }

  +

  + static void

  + change_decode_local(Bincode *binstat, Local *local)

  + {

  + uint64_t len;

  +

  + len = bincode_getu64(binstat);

  + local->path = bincode_getstr(binstat, len);

  + local->line = bincode_getu64(binstat);

  + }

  +

  + static void

  + change_decode_encoding(Bincode *binstat, char **encoding)

  + {

  + /* This is an optional field - basically just an Option<String> */

  + if (bincode_getu8(binstat)) {

  + uint64_t len = bincode_getu64(binstat);

  + *encoding = bincode_getstr(binstat, len);
• replacement in change.c at line 366
  [3.3797]→[3.3797:3802](∅→∅),[3.3802]→[2.2921:2959](∅→∅)
  − void

  − change_decode_hunks(Bincode *binstat)
  [3.3797]

  [3.3853]
  + static void

  + change_decode_hunks(Bincode *binstat, Hashed *hashed)
• edit in change.c at line 373
  [3.3911]

  [3.3911]
  + hunk_hunklistinit(&hashed->hunks, len);

  +
• replacement in change.c at line 376
  [3.3940]→[2.2960:3012](∅→∅),[2.3012]→[3.3998:4062](∅→∅),[3.3998]→[3.3998:4062](∅→∅)
  − basehunktype hunk_type = bincode_getu32(binstat);

  − printf("%d. %s\n", i + 1, hunk_basehunk_type_str(hunk_type));
  [3.3940]

  [3.4062]
  + Basehunk *bhunk = &hashed->hunks.entries[i];

  + bhunk->hunktype = bincode_getu32(binstat);

  + /* printf("%d. %s\n", i + 1, hunk_basehunk_type_str(bhunk->hunktype)); */
• replacement in change.c at line 380
  [3.4063]→[3.4063:4086](∅→∅)
  − switch (hunk_type) {
  [3.4063]

  [3.4086]
  + switch (bhunk->hunktype) {
• replacement in change.c at line 382
  [3.4099]→[3.4099:4304](∅→∅)
  − printf("Edit. change: Atom, local: Local, encoding: Option<Encoding>\n"

  − );

  − change_decode_atom(binstat); /* the change field */

  − /* change_decode_local */

  − /* change_decode_encoding (maybe) */
  [3.4099]

  [3.4304]
  + /* An edit is a (1) Atom (2) Local (3) Option<Encoding> */

  + change_decode_atom(

  + binstat, &bhunk->edit.change

  + ); /* the change field */

  + change_decode_local(binstat, &bhunk->edit.local);

  + change_decode_encoding(binstat, &bhunk->edit.encoding);

  + printf("%d. Edit in %s:%d\n", i + 1,

  + bhunk->edit.local.path, bhunk->edit.local.line);
• replacement in change.c at line 398
  [3.4491]→[3.4491:4531](∅→∅)
  − hunk_basehunk_type_str(hunk_type));
  [3.4491]

  [3.604]
  + hunk_basehunk_type_str(bhunk->hunktype));
• edit in change.c at line 401
  [3.620]→[3.4532:4599](∅→∅)
  − break; /* FIXME: Remove once full Hunk parsing is implemented */
• replacement in change.c at line 407
  [3.1079]→[3.1079:1083](∅→∅)
  − int
  [3.1079]

  [3.985]
  + static int
• edit in change.c at line 449
  [3.81]→[3.48:92](∅→∅)
  − /* dump_buf("hashed", buf, hashed_len); */
• replacement in change.c at line 501
  [3.76]→[3.4600:4675](∅→∅),[3.4675]→[3.128:179](∅→∅),[3.128]→[3.128:179](∅→∅)
  − uint8_t *bytes = xmalloc(len);

  − bincode_getbytes(&binstat, bytes, len);

  − dump_buf("metadata", bytes, len);

  − free(bytes);
  [3.76]

  [3.179]
  + hashed->metadata = xmalloc(len);

  + bincode_getbytes(&binstat, hashed->metadata, len);
• edit in change.c at line 505
  [3.183]→[3.848:894](∅→∅)
  − dump_buf("buf", binstat.buf, binstat.avail);
• replacement in change.c at line 506
  [3.278]→[3.4676:4708](∅→∅)
  − change_decode_hunks(&binstat);
  [3.278]

  [3.278]
  + change_decode_hunks(&binstat, hashed);
• replacement in change.c at line 606
  [3.2072]→[2.3345:3391](∅→∅)
  − hashprint(&hashed.dependencies.entries[i]);
  [3.2072]

  [3.2119]
  + hashprintln(&hashed.dependencies.entries[i]);
• replacement in change.c at line 610
  [3.2185]→[2.3440:3484](∅→∅)
  − hashprint(&hashed.extraknown.entries[i]);
  [3.2185]

  [3.2231]
  + hashprintln(&hashed.extraknown.entries[i]);
• replacement in atom.h at line 16
  [2.4340]→[2.4340:4401](∅→∅)
  − Hash upcontext; /* FIXME Option<Hash> */

  − Hash downcontext;
  [2.4340]

  [3.5342]
  + uint8_t previous;

  + uint8_t flag;

  + Position from;

  + Vertex to;

  + Hash introducedby; /* FIXME Should be Option */

  + } Edge;

  +

  + typedef struct {

  + size_t len;

  + Edge *entries;

  + } Edgelist;

  +

  + typedef struct {

  + Positionlist upcontext;

  + Positionlist downcontext;
• replacement in atom.h at line 34
  [3.5446]→[3.5446:5530](∅→∅)
  − /* FIXME inode: Position<Option<Hash>> = struct { change: Option<Hash>, pos: } */
  [3.5446]

  [2.4402]
  + Position inode;
• edit in atom.h at line 38
  [2.4433]

  [2.4433]
  + Edgelist edges;

  + Position inode;