Add PackFS v0: statically linked in-process VFS implementing concept.md
Implements the core design: a mount table published as an atomically- swapped snapshot; mem/dir/pack/overlay backends; copy-on-write overlay with copy-up and whiteout deletion; a checksummed append journal; compaction with exact-duplicate elimination; single-writer/wait-free- reader concurrency with a structural/content write split; openat2/ Landlock path containment for dir mounts; and load-time pack integrity validation. Zero required third-party dependencies. Sanitizer testing (ASan/UBSan) caught and led to fixing a genuine heap-use-after-free in the snapshot-reclamation path: the textbook "load pointer, then increment its refcount" pattern left a gap a concurrent writer could free through. Closed with a small reclaim_gate rwlock, documented in internal.h and CLAUDE.md since it's a pattern every refcounted structure in the codebase now follows. zip/tar import/export backends, recommended in concept.md Section 11, will not be built — a permanent project decision recorded in CLAUDE.md since concept.md itself is frozen and cannot be edited to reflect it. Includes a runnable demo (examples/demo.c, `make demo`) exercising the library end to end and proving cross-run persistence through the pack file, plus open-source scaffolding: MIT license, README, CONTRIBUTING, and a CI workflow running the test suite under ASan/UBSan/TSan. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UqJpkdJ6Njnt1pw3CbghzB
This commit is contained in:
Claude Sonnet 5
+304
@@ -0,0 +1,304 @@
|
||||
/*
|
||||
* pack.c — on-disk pack format (Section 9), load-time integrity
|
||||
* validation (Section 7), and compaction's writer (Section 4.1, 4.3,
|
||||
* 5.3, 9.1, 9.2).
|
||||
*
|
||||
* On-disk layout (a concrete realization of the illustrative sketch in
|
||||
* Section 9, extended with the checksum field Section 7 requires):
|
||||
*
|
||||
* PackHeader (fixed size, 8-byte aligned)
|
||||
* blobs (index_offset - sizeof(PackHeader) bytes)
|
||||
* PackIndexEntry[index_count] at index_offset, sorted by name (9.1)
|
||||
* strings (each name NUL-terminated) at strings_offset
|
||||
*/
|
||||
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "internal.h"
|
||||
|
||||
#define PACK_VERSION 1
|
||||
|
||||
typedef struct PackHeader {
|
||||
char magic[4];
|
||||
uint32_t version;
|
||||
uint64_t index_offset;
|
||||
uint64_t index_count;
|
||||
uint64_t strings_offset;
|
||||
uint64_t strings_len;
|
||||
uint64_t checksum; /* FNV-1a64 over [index_offset, strings_offset+strings_len) */
|
||||
} PackHeader;
|
||||
|
||||
static uint64_t align8(uint64_t n) { return (n + 7u) & ~(uint64_t)7u; }
|
||||
|
||||
/* ---- loading + validation (Section 7) ---- */
|
||||
|
||||
void pack_close(Pack *p) {
|
||||
if (!p) return;
|
||||
if (p->map && p->map != MAP_FAILED) munmap(p->map, p->map_len);
|
||||
if (p->fd >= 0) close(p->fd);
|
||||
free(p->path);
|
||||
free(p);
|
||||
}
|
||||
|
||||
int pack_load(const char *path, Pack **out, int *err) {
|
||||
int fd = open(path, O_RDONLY | O_CLOEXEC);
|
||||
if (fd < 0) { if (err) *err = VFS_ERR_NOENT; return -1; }
|
||||
|
||||
struct stat st;
|
||||
if (fstat(fd, &st) < 0) { close(fd); if (err) *err = VFS_ERR_IO; return -1; }
|
||||
size_t len = (size_t)st.st_size;
|
||||
if (len < sizeof(PackHeader)) { close(fd); if (err) *err = VFS_ERR_CORRUPT; return -1; }
|
||||
|
||||
void *map = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0);
|
||||
if (map == MAP_FAILED) { close(fd); if (err) *err = VFS_ERR_IO; return -1; }
|
||||
|
||||
const PackHeader *hdr = (const PackHeader *)map;
|
||||
if (memcmp(hdr->magic, "PKFS", 4) != 0 || hdr->version != PACK_VERSION) {
|
||||
munmap(map, len); close(fd);
|
||||
if (err) *err = VFS_ERR_CORRUPT;
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Bounds-check every offset before trusting it (Section 7): the
|
||||
* index and strings regions must lie within the file and must not
|
||||
* overlap the header. */
|
||||
if (hdr->index_offset < sizeof(PackHeader) ||
|
||||
hdr->index_offset > len ||
|
||||
hdr->index_count > (len - hdr->index_offset) / sizeof(PackIndexEntry)) {
|
||||
munmap(map, len); close(fd);
|
||||
if (err) *err = VFS_ERR_CORRUPT;
|
||||
return -1;
|
||||
}
|
||||
uint64_t index_bytes = hdr->index_count * (uint64_t)sizeof(PackIndexEntry);
|
||||
uint64_t index_end = hdr->index_offset + index_bytes;
|
||||
if (hdr->strings_offset < index_end || hdr->strings_offset > len ||
|
||||
hdr->strings_len > len - hdr->strings_offset) {
|
||||
munmap(map, len); close(fd);
|
||||
if (err) *err = VFS_ERR_CORRUPT;
|
||||
return -1;
|
||||
}
|
||||
uint64_t strings_end = hdr->strings_offset + hdr->strings_len;
|
||||
if (strings_end > len) {
|
||||
munmap(map, len); close(fd);
|
||||
if (err) *err = VFS_ERR_CORRUPT;
|
||||
return -1;
|
||||
}
|
||||
|
||||
uint64_t checksum = pfs_fnv1a64((const char *)map + hdr->index_offset,
|
||||
(size_t)(strings_end - hdr->index_offset));
|
||||
if (checksum != hdr->checksum) {
|
||||
munmap(map, len); close(fd);
|
||||
if (err) *err = VFS_ERR_CORRUPT;
|
||||
return -1;
|
||||
}
|
||||
|
||||
const PackIndexEntry *entries = (const PackIndexEntry *)((const char *)map + hdr->index_offset);
|
||||
const char *strings_base = (const char *)map + hdr->strings_offset;
|
||||
|
||||
for (uint64_t i = 0; i < hdr->index_count; i++) {
|
||||
const PackIndexEntry *e = &entries[i];
|
||||
if (e->data_off < sizeof(PackHeader) || e->data_off > hdr->index_offset ||
|
||||
e->size > hdr->index_offset - e->data_off) {
|
||||
munmap(map, len); close(fd);
|
||||
if (err) *err = VFS_ERR_CORRUPT;
|
||||
return -1;
|
||||
}
|
||||
if (e->name_off > hdr->strings_len || e->name_len > hdr->strings_len - e->name_off) {
|
||||
munmap(map, len); close(fd);
|
||||
if (err) *err = VFS_ERR_CORRUPT;
|
||||
return -1;
|
||||
}
|
||||
/* Section 9's names are NUL-terminated in the strings region; a
|
||||
* missing terminator is treated as corruption, not tolerated. */
|
||||
if (strings_base[e->name_off + e->name_len] != '\0') {
|
||||
munmap(map, len); close(fd);
|
||||
if (err) *err = VFS_ERR_CORRUPT;
|
||||
return -1;
|
||||
}
|
||||
if (i > 0) {
|
||||
const PackIndexEntry *prev = &entries[i - 1];
|
||||
if (strcmp(strings_base + prev->name_off, strings_base + e->name_off) >= 0) {
|
||||
munmap(map, len); close(fd);
|
||||
if (err) *err = VFS_ERR_CORRUPT; /* Section 9.1 relies on sortedness */
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Pack *p = (Pack *)calloc(1, sizeof(Pack));
|
||||
if (!p) { munmap(map, len); close(fd); if (err) *err = VFS_ERR_NOSPC; return -1; }
|
||||
p->fd = fd;
|
||||
p->map = map;
|
||||
p->map_len = len;
|
||||
p->entries = entries;
|
||||
p->entry_count = (size_t)hdr->index_count;
|
||||
p->strings_base = strings_base;
|
||||
p->strings_len = (size_t)hdr->strings_len;
|
||||
p->path = strdup(path);
|
||||
*out = p;
|
||||
return 0;
|
||||
}
|
||||
|
||||
const char *pack_entry_name(const Pack *p, const PackIndexEntry *e) {
|
||||
return p->strings_base + e->name_off;
|
||||
}
|
||||
|
||||
const void *pack_entry_data(const Pack *p, const PackIndexEntry *e) {
|
||||
return (const char *)p->map + e->data_off;
|
||||
}
|
||||
|
||||
int pack_find(const Pack *p, const char *name, const PackIndexEntry **out) {
|
||||
size_t lo = 0, hi = p->entry_count;
|
||||
while (lo < hi) {
|
||||
size_t mid = lo + (hi - lo) / 2;
|
||||
int c = strcmp(pack_entry_name(p, &p->entries[mid]), name);
|
||||
if (c == 0) { *out = &p->entries[mid]; return 1; }
|
||||
if (c < 0) lo = mid + 1; else hi = mid;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t lower_bound_str(const Pack *p, const char *key) {
|
||||
size_t lo = 0, hi = p->entry_count;
|
||||
while (lo < hi) {
|
||||
size_t mid = lo + (hi - lo) / 2;
|
||||
if (strcmp(pack_entry_name(p, &p->entries[mid]), key) < 0) lo = mid + 1; else hi = mid;
|
||||
}
|
||||
return lo;
|
||||
}
|
||||
|
||||
void pack_range(const Pack *p, const char *prefix, size_t *lo_out, size_t *hi_out) {
|
||||
size_t lo = lower_bound_str(p, prefix);
|
||||
size_t plen = strlen(prefix);
|
||||
char *upper = (char *)malloc(plen + 2);
|
||||
memcpy(upper, prefix, plen);
|
||||
upper[plen] = (char)0x7F; /* > any valid path byte, per Section 9.1's argument */
|
||||
upper[plen + 1] = '\0';
|
||||
size_t hi = lower_bound_str(p, upper);
|
||||
free(upper);
|
||||
*lo_out = lo;
|
||||
*hi_out = hi;
|
||||
}
|
||||
|
||||
/* ---- writing (compaction target, Section 4.1 / 4.3 / 9.2) ---- */
|
||||
|
||||
static int pack_build_entry_cmp(const void *a, const void *b) {
|
||||
return strcmp(((const PackBuildEntry *)a)->name, ((const PackBuildEntry *)b)->name);
|
||||
}
|
||||
|
||||
typedef struct DedupSlot {
|
||||
uint64_t hash;
|
||||
uint64_t size;
|
||||
uint64_t data_off;
|
||||
} DedupSlot;
|
||||
|
||||
int pack_write(const char *path, const PackBuildEntry *in, size_t count, int *err) {
|
||||
PackBuildEntry *entries = NULL;
|
||||
if (count > 0) {
|
||||
entries = (PackBuildEntry *)malloc(count * sizeof(PackBuildEntry));
|
||||
if (!entries) { if (err) *err = VFS_ERR_NOSPC; return -1; }
|
||||
memcpy(entries, in, count * sizeof(PackBuildEntry));
|
||||
}
|
||||
qsort(entries, count, sizeof(PackBuildEntry), pack_build_entry_cmp);
|
||||
|
||||
/* Pass 1: compute blob region with exact-duplicate elimination
|
||||
* (Section 9.2) and the strings region size. */
|
||||
DedupSlot *slots = count ? (DedupSlot *)malloc(count * sizeof(DedupSlot)) : NULL;
|
||||
size_t slot_count = 0;
|
||||
uint64_t *data_off_for = count ? (uint64_t *)malloc(count * sizeof(uint64_t)) : NULL;
|
||||
uint64_t blob_cursor = align8(sizeof(PackHeader));
|
||||
uint64_t strings_total = 0;
|
||||
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
uint64_t h = pfs_fnv1a64(entries[i].data, (size_t)entries[i].size);
|
||||
uint64_t reuse = UINT64_MAX;
|
||||
for (size_t j = 0; j < slot_count; j++) {
|
||||
if (slots[j].hash == h && slots[j].size == entries[i].size) {
|
||||
reuse = slots[j].data_off;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (reuse != UINT64_MAX) {
|
||||
data_off_for[i] = reuse;
|
||||
} else {
|
||||
data_off_for[i] = blob_cursor;
|
||||
slots[slot_count].hash = h;
|
||||
slots[slot_count].size = entries[i].size;
|
||||
slots[slot_count].data_off = blob_cursor;
|
||||
slot_count++;
|
||||
blob_cursor += entries[i].size;
|
||||
}
|
||||
strings_total += strlen(entries[i].name) + 1;
|
||||
}
|
||||
free(slots);
|
||||
|
||||
uint64_t index_offset = align8(blob_cursor);
|
||||
uint64_t index_bytes = (uint64_t)count * sizeof(PackIndexEntry);
|
||||
uint64_t strings_offset = index_offset + index_bytes;
|
||||
uint64_t total_size = strings_offset + strings_total;
|
||||
|
||||
unsigned char *buf = (unsigned char *)calloc(1, (size_t)total_size);
|
||||
if (!buf) { free(entries); free(data_off_for); if (err) *err = VFS_ERR_NOSPC; return -1; }
|
||||
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
/* Deduplicated entries (Section 9.2) share a data_off; writing
|
||||
* the same bytes to it more than once is redundant but harmless. */
|
||||
memcpy(buf + data_off_for[i], entries[i].data, entries[i].size);
|
||||
}
|
||||
|
||||
PackIndexEntry *out_entries = (PackIndexEntry *)(buf + index_offset);
|
||||
uint64_t str_cursor = 0;
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
size_t nlen = strlen(entries[i].name);
|
||||
out_entries[i].name_off = str_cursor;
|
||||
out_entries[i].name_len = (uint32_t)nlen;
|
||||
out_entries[i].data_off = data_off_for[i];
|
||||
out_entries[i].size = entries[i].size;
|
||||
out_entries[i].mode = entries[i].mode;
|
||||
out_entries[i].mtime = entries[i].mtime;
|
||||
memcpy(buf + strings_offset + str_cursor, entries[i].name, nlen + 1);
|
||||
str_cursor += nlen + 1;
|
||||
}
|
||||
|
||||
PackHeader hdr;
|
||||
memset(&hdr, 0, sizeof(hdr));
|
||||
memcpy(hdr.magic, "PKFS", 4);
|
||||
hdr.version = PACK_VERSION;
|
||||
hdr.index_offset = index_offset;
|
||||
hdr.index_count = count;
|
||||
hdr.strings_offset = strings_offset;
|
||||
hdr.strings_len = strings_total;
|
||||
hdr.checksum = pfs_fnv1a64(buf + index_offset, (size_t)(strings_offset + strings_total - index_offset));
|
||||
memcpy(buf, &hdr, sizeof(hdr));
|
||||
|
||||
free(entries);
|
||||
free(data_off_for);
|
||||
|
||||
/* Atomic compaction (Section 4.3): write to a temp file, fsync, then
|
||||
* rename over the target. A failure here aborts compaction and
|
||||
* leaves the existing pack untouched. */
|
||||
char tmp_path[PFS_PATH_MAX];
|
||||
snprintf(tmp_path, sizeof(tmp_path), "%s.tmp", path);
|
||||
int fd = open(tmp_path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||
if (fd < 0) { free(buf); if (err) *err = VFS_ERR_IO; return -1; }
|
||||
|
||||
size_t written = 0;
|
||||
while (written < (size_t)total_size) {
|
||||
ssize_t w = write(fd, buf + written, (size_t)total_size - written);
|
||||
if (w < 0) { close(fd); free(buf); unlink(tmp_path); if (err) *err = VFS_ERR_IO; return -1; }
|
||||
written += (size_t)w;
|
||||
}
|
||||
free(buf);
|
||||
|
||||
if (fsync(fd) < 0) { close(fd); unlink(tmp_path); if (err) *err = VFS_ERR_IO; return -1; }
|
||||
close(fd);
|
||||
|
||||
if (rename(tmp_path, path) < 0) { unlink(tmp_path); if (err) *err = VFS_ERR_IO; return -1; }
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user