retoorandClaude Sonnet 5
684c6d95de
A audit for other instances of the file-index O(n^2) shape (fixed previously via the persistent treap) found two more real issues: 1. pack_write's Section 9.2 exact-duplicate elimination was a linear scan of every previously-seen (hash, size) pair per entry -- O(n^2) total, invisible in the existing benchmark because its identical-content test files made every scan match on the first comparison. Measured with unique content instead: 80,000 entries took 1.74s, with a 20,000->80,000 step showing 15.6x for a 4x-N step, matching O(n^2)'s 16x prediction. The same scan also trusted a (hash, size) match without ever comparing actual bytes -- a latent correctness bug, since FNV-1a64 is explicitly not collision-resistant. Fixed both at once with an open-addressing hash table (load factor 1/2, linear probing) plus a memcmp verification before ever reusing a data_off. Post-fix: 80,000 entries in 0.044s (39.6x faster), ratio drops to 3.35x (consistent with O(n)). Covered permanently by two new/extended tests: a white-box assertion in test_pack_overlay.c that duplicate-content entries share one data_off and distinct-content entries do not, and a new tests/test_pack_write_perf.c regression tripwire against 10,000 unique entries. 2. vfs.c's mount table uses the same full-array-copy-per-write pattern the file index used to, confirmed O(n^2) via a new bench/bench.c category (500/2,000/8,000 mounts, both 4x-N steps showing 15-20x). Deliberately NOT rewritten: mount points are created by a program's own source code, not workload-driven, so realistic mount counts never reach the scale that made the file index's O(n^2) a real problem. Documented with full reasoning in BENCH.md and CLAUDE.md rather than silently left as an undocumented gap. Also fixes a real CI gap the new tests exposed: ci.yml's sanitizer-build steps never passed -D_GNU_SOURCE when compiling test files (only the library .o's got it), which was harmless while no test included internal.h and became a link failure once two did (internal.h needs _GNU_SOURCE for pthread_rwlock_t). And documents, in CONTRIBUTING.md and CLAUDE.md, a sandbox flake observed directly during this work's own sanitizer runs: ASan/UBSan test binaries occasionally fail to start with AddressSanitizer:DEADLYSIGNAL (sometimes looping rather than exiting), non-deterministically hitting different unrelated binaries across runs -- a startup race, not a memory-safety bug, confirmed by clean passes on retry; sanitizer runs in such an environment should be timeout-wrapped. BENCH.md's "After" table and Appendix B are replaced with the current, complete 54-measurement bench/bench.c run (the original 45 plus the new mount-scaling category); the pre-fix 45-measurement "Before" table is kept as the historical record, per this project's documentation standard. Verified: make test (all 6 binaries, including the 2 new/changed), a clean make all, and repeated ASan+UBSan runs (0 real findings; the DEADLYSIGNAL flake above was observed and correctly distinguished from a real finding by re-running until a clean pass). TSan could not be run in this sandbox (pre-existing, documented environment limitation). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UqJpkdJ6Njnt1pw3CbghzB
179 lines
6.3 KiB
C
179 lines
6.3 KiB
C
/* test_pack_overlay.c — overlay backend: copy-up, whiteout deletion of a
|
|
* lower-layer entry (Section 4.2), compaction (Section 4.1/5.3),
|
|
* cross-restart durability via journal replay (Section 4.4), and pack
|
|
* integrity validation rejecting a corrupted file (Section 7). */
|
|
|
|
#include <fcntl.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <unistd.h>
|
|
|
|
#include "packfs.h"
|
|
#include "internal.h" /* white-box: verifies Section 9.2 dedup actually shares data_off, not just "doesn't crash" */
|
|
#include "test_harness.h"
|
|
|
|
static char *tmp_pack_path(void) {
|
|
static char path[512];
|
|
snprintf(path, sizeof(path), "/tmp/packfs_test_pack_%d.img", (int)getpid());
|
|
return path;
|
|
}
|
|
|
|
int main(void) {
|
|
char *pack_path = tmp_pack_path();
|
|
unlink(pack_path);
|
|
char jpath[600];
|
|
snprintf(jpath, sizeof(jpath), "%s.jnl", pack_path);
|
|
unlink(jpath);
|
|
|
|
/* --- phase 1: build an initial pack directly (as if shipped) --- */
|
|
Vfs *v = vfs_new();
|
|
Backend *mem = backend_mem_new();
|
|
int oerr = 0;
|
|
Backend *ov = backend_overlay_new(pack_path, mem, &oerr);
|
|
CHECK(ov != NULL);
|
|
CHECK_EQ_INT(vfs_mount(v, "/", ov), VFS_OK);
|
|
|
|
int err = 0;
|
|
VfsFile *f = vfs_open(v, "/a.txt", VFS_O_WRONLY | VFS_O_CREAT, &err);
|
|
CHECK(f != NULL);
|
|
CHECK_EQ_INT(vfs_write(f, "AAAA", 4), 4);
|
|
CHECK_EQ_INT(vfs_close(f), VFS_OK);
|
|
|
|
f = vfs_open(v, "/dup.txt", VFS_O_WRONLY | VFS_O_CREAT, &err);
|
|
CHECK(f != NULL);
|
|
CHECK_EQ_INT(vfs_write(f, "AAAA", 4), 4); /* same content as a.txt: exercises dedup (Section 9.2) */
|
|
CHECK_EQ_INT(vfs_close(f), VFS_OK);
|
|
|
|
f = vfs_open(v, "/b.txt", VFS_O_WRONLY | VFS_O_CREAT, &err);
|
|
CHECK(f != NULL);
|
|
CHECK_EQ_INT(vfs_write(f, "BBBBBB", 6), 6);
|
|
CHECK_EQ_INT(vfs_close(f), VFS_OK);
|
|
|
|
CHECK_EQ_INT(vfs_sync(v, "/"), VFS_OK); /* compaction */
|
|
|
|
/* Section 9.2 dedup, actually verified (not just exercised): /a.txt
|
|
* and /dup.txt have identical content and must share one data_off
|
|
* in the compacted pack; /b.txt has different content and must not
|
|
* share either's. Catches both "dedup silently stopped happening"
|
|
* (a real, previously-unasserted gap) and "dedup over-matched two
|
|
* different files onto the same bytes" (the hash-collision-without-
|
|
* memcmp-verification bug fixed alongside the O(n^2) compaction
|
|
* cost — see pack.c's DedupSlot comment). */
|
|
{
|
|
Pack *p = NULL; int lerr = 0;
|
|
CHECK_EQ_INT(pack_load(pack_path, &p, &lerr), 0);
|
|
if (p) {
|
|
const PackIndexEntry *ea = NULL, *edup = NULL, *eb = NULL;
|
|
CHECK(pack_find(p, "/a.txt", &ea));
|
|
CHECK(pack_find(p, "/dup.txt", &edup));
|
|
CHECK(pack_find(p, "/b.txt", &eb));
|
|
if (ea && edup) CHECK_EQ_INT(ea->data_off, edup->data_off);
|
|
if (ea && eb) CHECK(ea->data_off != eb->data_off);
|
|
pack_close(p);
|
|
}
|
|
}
|
|
|
|
/* read-only open straight from the freshly-compacted pack */
|
|
f = vfs_open(v, "/a.txt", VFS_O_RDONLY, &err);
|
|
CHECK(f != NULL);
|
|
char buf[16] = {0};
|
|
CHECK_EQ_INT(vfs_read(f, buf, sizeof(buf)), 4);
|
|
CHECK_STR_EQ(buf, "AAAA");
|
|
CHECK_EQ_INT(vfs_close(f), VFS_OK);
|
|
|
|
/* --- copy-up on write intent, whiteout on delete (Section 4.2) --- */
|
|
f = vfs_open(v, "/a.txt", VFS_O_RDWR, &err); /* triggers copy-up */
|
|
CHECK(f != NULL);
|
|
CHECK_EQ_INT(vfs_write(f, "ZZZZ", 4), 4);
|
|
CHECK_EQ_INT(vfs_close(f), VFS_OK);
|
|
f = vfs_open(v, "/a.txt", VFS_O_RDONLY, &err);
|
|
memset(buf, 0, sizeof(buf));
|
|
CHECK_EQ_INT(vfs_read(f, buf, sizeof(buf)), 4);
|
|
CHECK_STR_EQ(buf, "ZZZZ");
|
|
CHECK_EQ_INT(vfs_close(f), VFS_OK);
|
|
|
|
CHECK_EQ_INT(vfs_unlink(v, "/b.txt"), VFS_OK); /* whiteout: b.txt only existed in the pack */
|
|
VfsStat st;
|
|
CHECK_EQ_INT(vfs_stat(v, "/b.txt", &st), VFS_ERR_NOENT);
|
|
|
|
/* --- durability: a fresh Vfs replays the journal without compacting --- */
|
|
vfs_unmount(v, "/");
|
|
backend_free(ov);
|
|
backend_free(mem);
|
|
vfs_free(v);
|
|
|
|
Vfs *v2 = vfs_new();
|
|
Backend *mem2 = backend_mem_new();
|
|
Backend *ov2 = backend_overlay_new(pack_path, mem2, &oerr);
|
|
CHECK(ov2 != NULL);
|
|
CHECK_EQ_INT(vfs_mount(v2, "/", ov2), VFS_OK);
|
|
|
|
CHECK_EQ_INT(vfs_stat(v2, "/b.txt", &st), VFS_ERR_NOENT); /* whiteout replayed */
|
|
f = vfs_open(v2, "/a.txt", VFS_O_RDONLY, &err);
|
|
CHECK(f != NULL);
|
|
memset(buf, 0, sizeof(buf));
|
|
CHECK_EQ_INT(vfs_read(f, buf, sizeof(buf)), 4);
|
|
CHECK_STR_EQ(buf, "ZZZZ"); /* the post-compaction write survived via the journal */
|
|
CHECK_EQ_INT(vfs_close(f), VFS_OK);
|
|
|
|
CHECK_EQ_INT(vfs_sync(v2, "/"), VFS_OK); /* compact again to fold the journal in */
|
|
vfs_unmount(v2, "/");
|
|
backend_free(ov2);
|
|
backend_free(mem2);
|
|
vfs_free(v2);
|
|
|
|
/* --- standalone read-only pack backend (Section 2.1/3.1), no overlay --- */
|
|
Vfs *v4 = vfs_new();
|
|
int perr = 0;
|
|
Backend *ro = backend_pack_new(pack_path, &perr);
|
|
CHECK(ro != NULL);
|
|
CHECK_EQ_INT(vfs_mount(v4, "/", ro), VFS_OK);
|
|
|
|
f = vfs_open(v4, "/a.txt", VFS_O_RDONLY, &err);
|
|
CHECK(f != NULL);
|
|
memset(buf, 0, sizeof(buf));
|
|
CHECK_EQ_INT(vfs_read(f, buf, sizeof(buf)), 4);
|
|
CHECK_STR_EQ(buf, "ZZZZ");
|
|
CHECK_EQ_INT(vfs_close(f), VFS_OK);
|
|
|
|
/* read-only: every mutation is rejected, not silently absorbed */
|
|
CHECK_EQ_INT(vfs_mkdir(v4, "/nope"), VFS_ERR_PERM);
|
|
CHECK_EQ_INT(vfs_unlink(v4, "/a.txt"), VFS_ERR_PERM);
|
|
f = vfs_open(v4, "/new.txt", VFS_O_WRONLY | VFS_O_CREAT, &err);
|
|
CHECK(f == NULL);
|
|
CHECK_EQ_INT(err, VFS_ERR_PERM);
|
|
CHECK_EQ_INT(vfs_sync(v4, "/"), VFS_ERR_PERM);
|
|
|
|
VfsDir rdir;
|
|
CHECK_EQ_INT(vfs_readdir(v4, "/", &rdir), VFS_OK);
|
|
CHECK(rdir.count > 0);
|
|
vfs_dir_free(&rdir);
|
|
|
|
vfs_unmount(v4, "/");
|
|
backend_free(ro);
|
|
vfs_free(v4);
|
|
|
|
/* --- pack integrity validation (Section 7): a corrupted pack is rejected --- */
|
|
int fd = open(pack_path, O_RDWR);
|
|
CHECK(fd >= 0);
|
|
unsigned char one;
|
|
CHECK_EQ_INT(pread(fd, &one, 1, 40), 1); /* somewhere inside the header/offsets region */
|
|
one = (unsigned char)(one ^ 0xFF);
|
|
CHECK_EQ_INT(pwrite(fd, &one, 1, 40), 1);
|
|
close(fd);
|
|
|
|
Vfs *v3 = vfs_new();
|
|
Backend *mem3 = backend_mem_new();
|
|
int oerr3 = 0;
|
|
Backend *ov3 = backend_overlay_new(pack_path, mem3, &oerr3);
|
|
CHECK(ov3 == NULL);
|
|
CHECK_EQ_INT(oerr3, VFS_ERR_CORRUPT);
|
|
backend_free(mem3);
|
|
vfs_free(v3);
|
|
|
|
unlink(pack_path);
|
|
unlink(jpath);
|
|
TEST_MAIN_END();
|
|
}
|