diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f97f64601..373caf4f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -324,6 +324,45 @@ jobs: fi echo "ok: all source files carry the header" + # convert.py has two trunk paths and only the --reclaim one called + # build_engram_meta(), so a default DeepSeek-V4.1 conversion wrote the + # 110 GB of Engram tables and none of the index that addresses them. + # The container then failed to open with five words and no subject, + # because engram_open() was also the one -2 in the load path that + # printed nothing (#71) — six and a half hours of conversion to find + # out. No test that lacks real weights can see either half, so the + # invariant is checked on the source instead. + # + # It is checked per BLOCK and not by counting call sites, because the + # bug shipped alongside a duplicated build_engram_meta() on the other + # path: two tables, two indexes, totals equal, and the default path + # still wrote nothing. A counting guard passes on the broken tree. + - name: every block writing Engram tables also writes the index + run: | + python3 - <<'EOF' + import ast, sys + tree = ast.parse(open("tools/convert.py").read()) + def names(stmt): + return {n.func.id for n in ast.walk(stmt) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)} + bad = [] + for node in ast.walk(tree): + for field in ("body", "orelse", "finalbody"): + block = getattr(node, field, None) + if not isinstance(block, list): + continue + called = set().union(*(names(s) for s in block)) if block else set() + if "build_engram" in called and "build_engram_meta" not in called: + bad.append(min(s.lineno for s in block)) + for line in sorted(set(bad)): + print(f"FAIL: block at tools/convert.py:{line} calls build_engram() " + f"but not build_engram_meta()") + if bad: + print("A container whose Engram tables have no engram.json cannot be opened.") + sys.exit(1) + print("ok: every block that writes Engram tables also writes the index") + EOF + # The cross-build job proves the code compiles for Windows and the run job # proves those binaries execute. Neither builds *on* Windows, so neither # ever ran the Makefile's platform detection, the shell harness, or the diff --git a/src/model.c b/src/model.c index ad3c8fe81..88394988c 100644 --- a/src/model.c +++ b/src/model.c @@ -2452,7 +2452,16 @@ static int engram_open(waste_model *m, const char *dir, const js_doc *d) snprintf(path, sizeof path, "%s/engram.json", dir); char *es = slurp(path, NULL); - if (!es) return -2; + /* Name the file. This was the one -2 in the whole load path that printed + * nothing, and DeepSeek-V4.1 is the only architecture that can reach it, + * so a container missing its Engram index failed with five words and no + * subject: "open: malformed container". Every other WASTE_E_FORMAT site + * says what it did not like. */ + if (!es) { + fprintf(stderr, "waste: %s is missing — the Engram tables are there " + "but the index that addresses them is not\n", path); + return -2; + } js_doc ed; if (js_parse(&ed, es) < 0) { free(es); return -2; } diff --git a/tools/convert.py b/tools/convert.py index d04cfd777..785732c25 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -2045,7 +2045,6 @@ def layer_source_ok(L): if ds41: engram = build_engram(st, args.out, cfg, args.engram_bits) build_engram_meta(args.src, cfg, args.out) - build_engram_meta(args.src, cfg, args.out) reclaim(debt, args.reclaim, ShardDebt.TRUNK, "trunk") # A resumed conversion is holding the shards of every layer an @@ -2443,6 +2442,7 @@ def reclaim_layer(L, size): reclaim_ple_if_complete(debt, args.reclaim, tindex) if ds41 and not engram: engram = build_engram(st, args.out, cfg, args.engram_bits) + build_engram_meta(args.src, cfg, args.out) trunk_path = os.path.join(args.out, "trunk.bin") trunk_tmp = trunk_path + ".tmp"