laumann/ani - Change QSQNGA5KZ4BVWE4FUJUMOP6SJRIXAC2PYU6JSRG5KXWLYWZZMAJQC

changestore: load changes on demand

Add a changestoreload() that checks the capacity of the current changestore, and loads in a new change by its hash.

This allows loading changes dynamically as needed so there is no need to preload anything. On our worstcase (check2) test target, the memory usage drops from 140MB to around 50MB, and the number allocations drop from 1204 to 326.

The xrealloc() function cannot zero out new memory, which can cause segfaults because garbage data is treated as non-NULL pointer values and passed to free(). To work around this, we initialize optional values to NULL as needed.

Created by laumann on July 20, 2023

QSQNGA5KZ4BVWE4FUJUMOP6SJRIXAC2PYU6JSRG5KXWLYWZZMAJQC

Dependencies

In channels

main

In discussions

#7 changestore: allow dynamically expanding the list of loaded changes

Change contents

Insertion in change.h at line 106 [9.80]

[8.1363]

	char *repodir; /* absolute path to the changestore */

Replacement in change.h at line 112 [9.80]

B:BD[8.1423] → [8.1423:1479]

void changestoreinit(struct changestore *, size_t cap);

[8.1423]

[8.1479]

void changestoreinit(struct changestore *, size_t cap, const char *);

Insertion in change.h at line 120 [9.80]

[8.1690]

[9.1005]

int changestoreload(struct changestore *, struct hash *);

Insertion in change.c at line 454 [9.1047]
[10.1130]
[11.5467]
```
	hashed->header.description = NULL;
```
Insertion in change.c at line 499 [9.1047]
[11.6005]
[12.60]
```
	hashed->metadata = NULL;
```

Insertion in change.c at line 574 [9.1047]

[2.594]

[8.3153]

			if (!ch) {
				changestoreload(changes, &vto->change);
				ch = changestoreget(changes, &vto->change);
			}

Replacement in change.c at line 692 [9.1047]

B:BD[13.635] → [4.690:745]

		print_positionlist(&v->downcontext);
		printf("\n");

[13.635]

[13.690]

		/* print_positionlist(&v->downcontext); */
		/* printf("\n"); */

Deletion in change.c at line 903 [9.1047]

B:BD[14.1849] → [14.1849:1853]

B:BD[14.1853] → [8.5529:5967]

B:BD[8.5991] → [8.5991:6077]

B:BD[8.6077] → [7.38:67]

∅:D[7.67] → [8.6117:6184]

B:BD[8.6117] → [8.6117:6184]

∅:D[14.1853] → [13.992:995]

∅:D[8.6184] → [13.992:995]

B:BD[13.992] → [13.992:995]

B:BD[13.995] → [6.0:19]

∅:D[6.19] → [8.6185:6193]

B:BD[13.995] → [8.6185:6193]

B:BD[8.6193] → [7.68:319]

		}
	}
}
struct change *
changestoreget(struct changestore *store, struct hash *hash)
{
	size_t i;
	for (i = 0; i < store->len; i++) {
		if (hasheq(hash, &store->entries[i].hash))
			return &store->entries[i].change;
	}
	return NULL;
}
void
changestoreinit(struct changestore *s, size_t cap)
{
	s->entries = xmalloc(sizeof(struct changeentry) * cap);
	s->cap = cap;
	s->len = 0;
}
void
changestorefree(struct changestore *s)
{
	size_t i;
	struct change *c;
	/* Free all the entries first */
	for (i = 0; i < s->len; i++) {
		c = &s->entries[i].change;
		hashedfree(&c->hashed);
		if (c->contents)
			free(c->contents);
	}
	free(s->entries);
}
void
authorsfree(struct authors *authors)
{
	size_t i, j;
	struct author *author;
	for (i = 0; i < authors->len; i++) {
		author = &authors->map[i];
		for (j = 0; j < author->len; j++) {
			free(author->entries[j].key);
			free(author->entries[j].value);

Deletion in change.c at line 904 [9.1047]
B:BD[7.323] → [7.323:348]
```
		free(author->entries);
```
Deletion in change.c at line 905 [9.1047]
B:BD[7.351] → [7.351:372]
```
	free(authors->map);
```
Deletion in change.c at line 907 [9.1047]
B:BD[7.375] → [7.375:380]
∅:D[7.380] → [8.6193:6224]
B:BD[8.6193] → [8.6193:6224]
B:BD[8.6224] → [7.381:529]
∅:D[7.529] → [8.6224:6329]
B:BD[8.6224] → [8.6224:6329]
B:BD[8.6329] → [5.2198:2224]
∅:D[5.2224] → [13.995:997]
∅:D[8.6329] → [13.995:997]
B:BD[13.995] → [13.995:997]
B:BD[13.997] → [14.1854:1855]
```
void
hashedfree(struct hashed *h)
{
	free(h->header.message);
	if (h->header.description)
		free(h->header.description);
	free(h->header.timestamp);
	authorsfree(&h->header.authors);
	hashlist_free(&h->dependencies);
	hashlist_free(&h->extraknown);
	if (h->metadata)
		free(h->metadata);
	hunklistfree(&h->hunks);
}
```

Insertion in change.c at line 1015 [9.1047]

[8.8048]

}
struct change *
changestoreget(struct changestore *store, struct hash *hash)
{
	size_t i;
	for (i = 0; i < store->len; i++) {
		if (hasheq(hash, &store->entries[i].hash))
			return &store->entries[i].change;
	}
	return NULL;
}
// loading changes as needed: ==15877==   total heap usage: 326 allocs, 326 frees, 50,066,715 bytes allocated
/**
 * load a change by the given hash into the changestore
 */
int
changestoreload(struct changestore *store, struct hash *hash)
{
	struct changeentry *ch;
	size_t i;
	int err;
	if (store->len == store->cap) {
		/* resize */
		size_t newcap = store->cap << 1;
		store->entries = xrealloc(
			store->entries, sizeof(struct changeentry) * newcap
		);
		store->cap = newcap;
	}
	i = store->len;
	ch = &store->entries[i];
	err = loadchangeh(&ch->change, &ch->hash, store->repodir, hash, 0);
	if (err)
		return err;
	store->len++;
	return err;
}
void
changestoreinit(struct changestore *s, size_t cap, const char *repodir)
{
	s->repodir = xstrdup(repodir);
	s->entries = xmalloc(sizeof(struct changeentry) * cap);
	s->cap = cap;
	s->len = 0;
}
void
changestorefree(struct changestore *s)
{
	size_t i;
	struct change *c;
	/* Free all the entries first */
	for (i = 0; i < s->len; i++) {
		c = &s->entries[i].change;
		hashedfree(&c->hashed);
		if (c->contents)
			free(c->contents);
	}
	free(s->repodir);
	free(s->entries);
}
void
authorsfree(struct authors *authors)
{
	size_t i, j;
	struct author *author;
	for (i = 0; i < authors->len; i++) {
		author = &authors->map[i];
		for (j = 0; j < author->len; j++) {
			free(author->entries[j].key);
			free(author->entries[j].value);
		}
		free(author->entries);
	}
	free(authors->map);

Insertion in change.c at line 1103 [9.1047]

[8.8051]

void
hashedfree(struct hashed *h)
{
	free(h->header.message);
	if (h->header.description)
		free(h->header.description);
	free(h->header.timestamp);
	authorsfree(&h->header.authors);
	hashlist_free(&h->dependencies);
	hashlist_free(&h->extraknown);
	if (h->metadata)
		free(h->metadata);
	hunklistfree(&h->hunks);
}

Replacement in change.c at line 1130 [9.1047]
B:BD[8.8293] → [8.8293:8329]
```
	changestoreinit(&changestore, 32);
```
[8.8293]
[8.8329]
```
	changestoreinit(&changestore, 4, repodir);
```

Deletion in change.c at line 1139 [9.1047]

B:BD[8.8516] → [8.8516:8517]

B:BD[8.8517] → [8.8517:8786]

B:BD[8.8786] → [3.27:72]

∅:D[3.72] → [8.8822:9090]

B:BD[8.8822] → [8.8822:9090]

B:BD[8.9090] → [3.73:118]

∅:D[3.118] → [8.9126:9129]

B:BD[8.9126] → [8.9126:9129]


	for (i = 0; i < ch->change.hashed.dependencies.len; i++) {
		err = loadchangeh(
			&changestore.entries[x].change,
			&changestore.entries[x].hash, repodir,
			&ch->change.hashed.dependencies.entries[i], verbose
		);
		if (err)
			goto changeout;
		changestore.len++;
		changestore.entries[x].num = x + 1;
		x++;
	}
	for (i = 0; i < ch->change.hashed.extraknown.len; i++) {
		err = loadchangeh(
			&changestore.entries[x].change,
			&changestore.entries[x].hash, repodir,
			&ch->change.hashed.extraknown.entries[i], verbose
		);
		if (err)
			goto changeout;
		changestore.len++;
		changestore.entries[x].num = x + 1;
		x++;
	}