laumann/ani - Change HMFIDMVZCIROCMAWQWVWWUULOW3ZHKMLRMOPAE4QUDLVR437O2OAC

changestore: load changes on demand

Add a changestoreload() that checks the capacity of the current changestore, and loads in a new change by its hash.

This allows loading changes dynamically as needed so there is no need to preload anything. On our worstcase (check2) test target, the memory usage drops from 140MB to around 50MB, and the number allocations drop from 1204 to 326.

The xrealloc() function cannot zero out new memory, which can cause segfaults because garbage data is treated as non-NULL pointer values and passed to free(). To work around this, we initialize optional values to NULL as needed.

Created by laumann on July 20, 2023

HMFIDMVZCIROCMAWQWVWWUULOW3ZHKMLRMOPAE4QUDLVR437O2OAC

Dependencies

In discussions

#7 changestore: allow dynamically expanding the list of loaded changes

Change contents

Insertion in change.h at line 119 [8.80]

[7.1690]

[8.1005]

int changestoreload(struct changestore *, struct hash *, const char *);

Insertion in change.c at line 454 [8.1047]
[9.1130]
[10.5467]
```
	hashed->header.description = NULL;
```
Insertion in change.c at line 499 [8.1047]
[10.6005]
[11.60]
```
	hashed->metadata = NULL;
```
Replacement in change.c at line 635 [8.1047]
B:BD[7.2080] → [7.2080:2091]
```
	size_t *n
```
[7.2080]
[7.2091]
```
	const char *repodir, size_t *n
```

Insertion in change.c at line 669 [8.1047]

[2.594]

[7.3153]

			if (!ch) {
				changestoreload(changes, &vto->change, repodir);
				ch = changestoreget(changes, &vto->change);
			}

Replacement in change.c at line 770 [8.1047]
B:BD[3.674] → [3.674:687]
```
	int verbose
```
[3.674]
[3.687]
```
	const char *repodir, int verbose
```

Replacement in change.c at line 800 [8.1047]

B:BD[7.4983] → [7.4983:5033]

		res = changecontents(changes, a, contents, &n);

[7.4697]

[2.672]

		// valgrind --leak-check=full build/ani change YM2LC6QP2D7TK3JUKOXWUWTQEGMCEGQDGPTOZPWC3BRDFNRKSZWQC # this one has file additions
		// our current scheme allocates more than 140 MB!! See about bringing that down
		// the "scratch pad" idea only takes out about four allocations here...
		// ==14997==   total heap usage: 1,204 allocs, 1,204 frees, 149,446,210 bytes allocated
		res = changecontents(changes, a, contents, repodir, &n);

Replacement in change.c at line 839 [8.1047]
B:BD[3.854] → [3.854:867]
```
	int verbose
```
[3.854]
[7.5206]
```
	const char *repodir, int verbose
```

Replacement in change.c at line 883 [8.1047]

B:BD[12.1554] → [3.868:914]

			print_atom(changes, c, contents, verbose);

[12.1554]

[12.1582]

			print_atom(changes, c, contents, repodir, verbose);

Replacement in change.c at line 896 [8.1047]

B:BD[7.5303] → [3.915:961]

			print_atom(changes, c, contents, verbose);

[7.5303]

[7.5340]

			print_atom(changes, c, contents, repodir, verbose);

Replacement in change.c at line 898 [8.1047]

B:BD[7.5341] → [3.962:1018]

			print_atom(changes, replacement, contents, verbose);

[7.5341]

[12.1779]

			print_atom(
				changes, replacement, contents, repodir, verbose
			);

Replacement in change.c at line 925 [8.1047]

B:BD[13.551] → [3.1019:1076]

			print_atom(changes, &f->contents, contents, verbose);

[13.551]

[13.552]

			print_atom(
				changes, &f->contents, contents, repodir,
				verbose
			);

Replacement in change.c at line 1002 [8.1047]

B:BD[14.1206] → [3.1077:1130]

			print_atom(changes, &r->name, contents, verbose);

[14.1206]

[14.1241]

			print_atom(
				changes, &r->name, contents, repodir, verbose
			);

Replacement in change.c at line 1006 [8.1047]

B:BD[14.1263] → [3.1131:1185]

			print_atom(changes, &r->inode, contents, verbose);

[14.1263]

[14.1299]

			print_atom(
				changes, &r->inode, contents, repodir, verbose
			);

Deletion in change.c at line 1014 [8.1047]

B:BD[12.1849] → [12.1849:1853]

B:BD[12.1853] → [7.5529:5967]

B:BD[7.5991] → [7.5991:6077]

B:BD[7.6077] → [6.38:67]

∅:D[6.67] → [7.6117:6184]

B:BD[7.6117] → [7.6117:6184]

∅:D[12.1853] → [15.992:995]

∅:D[7.6184] → [15.992:995]

B:BD[15.992] → [15.992:995]

B:BD[15.995] → [5.0:19]

∅:D[5.19] → [7.6185:6193]

B:BD[15.995] → [7.6185:6193]

B:BD[7.6193] → [6.68:319]

		}
	}
}
struct change *
changestoreget(struct changestore *store, struct hash *hash)
{
	size_t i;
	for (i = 0; i < store->len; i++) {
		if (hasheq(hash, &store->entries[i].hash))
			return &store->entries[i].change;
	}
	return NULL;
}
void
changestoreinit(struct changestore *s, size_t cap)
{
	s->entries = xmalloc(sizeof(struct changeentry) * cap);
	s->cap = cap;
	s->len = 0;
}
void
changestorefree(struct changestore *s)
{
	size_t i;
	struct change *c;
	/* Free all the entries first */
	for (i = 0; i < s->len; i++) {
		c = &s->entries[i].change;
		hashedfree(&c->hashed);
		if (c->contents)
			free(c->contents);
	}
	free(s->entries);
}
void
authorsfree(struct authors *authors)
{
	size_t i, j;
	struct author *author;
	for (i = 0; i < authors->len; i++) {
		author = &authors->map[i];
		for (j = 0; j < author->len; j++) {
			free(author->entries[j].key);
			free(author->entries[j].value);

Deletion in change.c at line 1015 [8.1047]
B:BD[6.323] → [6.323:348]
```
		free(author->entries);
```

Deletion in change.c at line 1016 [8.1047]

B:BD[6.351] → [6.351:380]

∅:D[6.380] → [7.6193:6224]

B:BD[7.6193] → [7.6193:6224]

B:BD[7.6224] → [6.381:529]

∅:D[6.529] → [7.6224:6329]

B:BD[7.6224] → [7.6224:6329]

B:BD[7.6329] → [4.2198:2224]

	free(authors->map);
}
void
hashedfree(struct hashed *h)
{
	free(h->header.message);
	if (h->header.description)
		free(h->header.description);
	free(h->header.timestamp);
	authorsfree(&h->header.authors);
	hashlist_free(&h->dependencies);
	hashlist_free(&h->extraknown);
	if (h->metadata)
		free(h->metadata);
	hunklistfree(&h->hunks);

Insertion in change.c at line 1126 [8.1047]

[7.8048]

}
struct change *
changestoreget(struct changestore *store, struct hash *hash)
{
	size_t i;
	for (i = 0; i < store->len; i++) {
		if (hasheq(hash, &store->entries[i].hash))
			return &store->entries[i].change;
	}
	return NULL;
}
// loading changes as needed: ==15877==   total heap usage: 326 allocs, 326 frees, 50,066,715 bytes allocated
/**
 * load a change by the given hash into the changestore
 */
int
changestoreload(
	struct changestore *store, struct hash *hash, const char *repodir
)
{
	struct changeentry *ch;
	size_t i;
	int err;
	if (store->len == store->cap) {
		printf("resizing... len = %lu cap = %lu\n", store->len,
		       store->cap);
		/* resize */
		size_t newcap = store->cap << 1;
		store->entries = xrealloc(
			store->entries, sizeof(struct changeentry) * newcap
		);
		store->cap = newcap;
	}
	printf("len = %lu cap = %lu\n", store->len, store->cap);
	i = store->len;
	ch = &store->entries[i];
	err = loadchangeh(&ch->change, &ch->hash, repodir, hash, 0);
	if (err)
		return err;
	store->len++;
	return err;
}
void
changestoreinit(struct changestore *s, size_t cap)
{
	s->entries = xmalloc(sizeof(struct changeentry) * cap);
	s->cap = cap;
	s->len = 0;
}
void
changestorefree(struct changestore *s)
{
	size_t i;
	struct change *c;
	/* Free all the entries first */
	for (i = 0; i < s->len; i++) {
		c = &s->entries[i].change;
		hashedfree(&c->hashed);
		if (c->contents)
			free(c->contents);
	}
	free(s->entries);

Insertion in change.c at line 1200 [8.1047]

[7.8051]

void
authorsfree(struct authors *authors)
{
	size_t i, j;
	struct author *author;
	for (i = 0; i < authors->len; i++) {
		author = &authors->map[i];
		for (j = 0; j < author->len; j++) {
			free(author->entries[j].key);
			free(author->entries[j].value);
		}
		free(author->entries);
	}
	free(authors->map);
}
void
hashedfree(struct hashed *h)
{
	free(h->header.message);
	if (h->header.description)
		free(h->header.description);
	free(h->header.timestamp);
	authorsfree(&h->header.authors);
	hashlist_free(&h->dependencies);
	hashlist_free(&h->extraknown);
	if (h->metadata)
		free(h->metadata);
	hunklistfree(&h->hunks);
}

Replacement in change.c at line 1244 [8.1047]
B:BD[7.8293] → [7.8293:8329]
```
	changestoreinit(&changestore, 32);
```
[7.8293]
[7.8329]
```
	changestoreinit(&changestore, 4);
```
Insertion in change.c at line 1254 [8.1047]
[7.8517]
[7.8517]
```
	/*
```
Insertion in change.c at line 1279 [8.1047]
[7.9129]
[7.9129]
```
	*/
```

Replacement in change.c at line 1282 [8.1047]

B:BD[3.1201] → [3.1201:1266]

		&changestore, &ch->change.hashed, ch->change.contents, verbose

[3.1201]

[3.1266]

		&changestore, &ch->change.hashed, ch->change.contents, repodir,
		verbose

changestore: load changes on demand

Dependencies

In discussions

Change contents

Insertion in change.h at line 119 [8.80]

Insertion in change.c at line 454 [8.1047]

Insertion in change.c at line 499 [8.1047]

Replacement in change.c at line 635 [8.1047]

Insertion in change.c at line 669 [8.1047]

Replacement in change.c at line 770 [8.1047]