From ada1836639e8b272d3485f481bfb1ae8e12e2ac2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 20:25:28 +0800 Subject: [PATCH 1/3] fix(oci): convert layer whiteouts into overlayfs whiteouts mkfs.erofs --tar=f without --aufs keeps OCI whiteout entries (.wh., .wh..wh..opq) as plain empty files, so the guest overlayfs never hides the paths a later layer deletes: they reappear in every VM built from such an image, together with the .wh.* files themselves. Pass --aufs (available in the enforced erofs-utils >= 1.8) so whiteouts become 0:0 char devices and opaque markers become overlay opaque dirs. Blobs converted before this change are cached by layer digest and would keep being reused, so blob keys now hash the conversion format with the layer digest: pulls and imports miss the stale blobs and convert again, a pull of an unchanged manifest no longer counts as up to date, images and VMs still referencing old blobs keep working, and the existing image GC collects the old blobs once nothing references them. --- images/oci/erofs.go | 9 ++++ images/oci/erofs_test.go | 107 ++++++++++++++++++++++++++++++++++++++- images/oci/fetch.go | 16 +++++- images/oci/import.go | 2 +- images/oci/process.go | 11 ++-- images/oci/pull.go | 8 ++- images/oci/pull_test.go | 5 +- 7 files changed, 144 insertions(+), 14 deletions(-) diff --git a/images/oci/erofs.go b/images/oci/erofs.go index abf659fe..7c46b468 100644 --- a/images/oci/erofs.go +++ b/images/oci/erofs.go @@ -3,6 +3,8 @@ package oci import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "io" @@ -16,6 +18,7 @@ import ( const ( erofsBlockSize = 4096 erofsCompression = "lz4hc" + erofsBlobFormat = "aufs" // erofs-utils < 1.8 tar mode writes corrupt compressed clusters yet exits 0, surfacing only as EUCLEAN at read time (#94). erofsMinMajor = 1 @@ -38,6 +41,7 @@ func runErofsConversion(ctx context.Context, src io.Reader, scanDir, namePrefix, cmd := exec.CommandContext( //nolint:gosec ctx, "mkfs.erofs", "--tar=f", + "--aufs", fmt.Sprintf("-z%s", erofsCompression), fmt.Sprintf("-C%d", erofsBlockSize), "-T0", @@ -105,3 +109,8 @@ func erofsVersionAtLeast(output string) error { return fmt.Errorf("mkfs.erofs %s.%s is older than %d.%d: its tar mode silently corrupts layers (EUCLEAN at read time) — upgrade erofs-utils", m[1], m[2], erofsMinMajor, erofsMinMinor) } + +func blobHex(layerHex string) string { + sum := sha256.Sum256([]byte(erofsBlobFormat + ":" + layerHex)) + return hex.EncodeToString(sum[:]) +} diff --git a/images/oci/erofs_test.go b/images/oci/erofs_test.go index 101c753f..5b22a205 100644 --- a/images/oci/erofs_test.go +++ b/images/oci/erofs_test.go @@ -1,6 +1,19 @@ package oci -import "testing" +import ( + "archive/tar" + "bytes" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +const ( + erofsTypeReg = "1" + erofsTypeDir = "2" + erofsTypeChrdev = "3" +) func TestErofsVersionAtLeast(t *testing.T) { tests := []struct { @@ -25,3 +38,95 @@ func TestErofsVersionAtLeast(t *testing.T) { }) } } + +func TestBlobHex(t *testing.T) { + layerHex := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + got := blobHex(layerHex) + if got == layerHex { + t.Fatalf("blobHex(%s) reuses the layer digest as the blob key", layerHex) + } + if len(got) != len(layerHex) || strings.Trim(got, "0123456789abcdef") != "" { + t.Fatalf("blobHex(%s) = %q, want 64 lowercase hex chars", layerHex, got) + } + if again := blobHex(layerHex); again != got { + t.Fatalf("blobHex not deterministic: %s != %s", again, got) + } + if other := blobHex(strings.Repeat("f", 64)); other == got { + t.Fatalf("blobHex collides for distinct layers: %s", other) + } +} + +func TestRunErofsConversionWhiteouts(t *testing.T) { + if _, err := exec.LookPath("dump.erofs"); err != nil { + t.Skip("dump.erofs not installed") + } + if err := checkErofsVersion(t.Context()); err != nil { + t.Skipf("mkfs.erofs unusable: %v", err) + } + + var layer bytes.Buffer + tw := tar.NewWriter(&layer) + for _, hdr := range []*tar.Header{ + {Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755}, + {Name: "a/.wh.x", Typeflag: tar.TypeReg, Mode: 0o644}, + {Name: ".wh.b", Typeflag: tar.TypeReg, Mode: 0o644}, + {Name: "c/", Typeflag: tar.TypeDir, Mode: 0o755}, + {Name: "c/.wh..wh..opq", Typeflag: tar.TypeReg, Mode: 0o644}, + {Name: "c/keep", Typeflag: tar.TypeReg, Mode: 0o644}, + } { + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("write tar header %s: %v", hdr.Name, err) + } + } + if err := tw.Close(); err != nil { + t.Fatalf("close tar: %v", err) + } + + dir := t.TempDir() + image := filepath.Join(dir, "layer.erofs") + if _, _, err := runErofsConversion(t.Context(), &layer, dir, "layer", "00000000-0000-5000-8000-000000000000", image); err != nil { + t.Fatalf("runErofsConversion: %v", err) + } + + tests := []struct { + path string + want map[string]string + }{ + {"/", map[string]string{"a": erofsTypeDir, "b": erofsTypeChrdev, "c": erofsTypeDir}}, + {"/a", map[string]string{"x": erofsTypeChrdev}}, + {"/c", map[string]string{"keep": erofsTypeReg}}, + } + for _, tt := range tests { + got := listErofsDir(t, image, tt.path) + if len(got) != len(tt.want) { + t.Errorf("%s entries = %v, want %v", tt.path, got, tt.want) + continue + } + for name, typ := range tt.want { + if got[name] != typ { + t.Errorf("%s type = %q, want %q (entries %v)", filepath.Join(tt.path, name), got[name], typ, got) + } + } + } +} + +func listErofsDir(t *testing.T, image, path string) map[string]string { + t.Helper() + out, err := exec.CommandContext(t.Context(), "dump.erofs", "--ls", "--path="+path, image).CombinedOutput() + if err != nil { + t.Fatalf("dump.erofs --ls %s: %v (output: %s)", path, err, out) + } + _, listing, ok := strings.Cut(string(out), "FILENAME") + if !ok { + t.Fatalf("dump.erofs --ls %s: no listing in %q", path, out) + } + entries := map[string]string{} + for line := range strings.Lines(listing) { + fields := strings.Fields(line) + if len(fields) != 3 || fields[2] == "." || fields[2] == ".." { + continue + } + entries[fields[2]] = fields[1] + } + return entries +} diff --git a/images/oci/fetch.go b/images/oci/fetch.go index c5d7f50a..10727183 100644 --- a/images/oci/fetch.go +++ b/images/oci/fetch.go @@ -59,9 +59,21 @@ func fetchImage(ctx context.Context, imageRef string) (ref, digestHex string, la return ref, digestHex, layers, nil } -func isUpToDate(conf *Config, idx *imageIndex, ref, digestHex string) bool { +func layerBlobHexes(layers []v1.Layer) ([]string, error) { + hexes := make([]string, len(layers)) + for i, layer := range layers { + d, err := layer.Digest() + if err != nil { + return nil, fmt.Errorf("get layer %d digest: %w", i, err) + } + hexes[i] = blobHex(d.Hex) + } + return hexes, nil +} + +func isUpToDate(conf *Config, idx *imageIndex, ref, digestHex string, blobHexes []string) bool { entry, ok := idx.Images[ref] - if !ok || entry == nil || entry.ManifestDigest != images.NewDigest(digestHex) { + if !ok || entry == nil || entry.ManifestDigest != images.NewDigest(digestHex) || !slices.Equal(entry.DigestHexes(), blobHexes) { return false } if !utils.ValidFile(conf.KernelPath(entry.KernelLayer.Hex())) || diff --git a/images/oci/import.go b/images/oci/import.go index 5f276350..fdf99dd5 100644 --- a/images/oci/import.go +++ b/images/oci/import.go @@ -114,7 +114,7 @@ func processTarReader(ctx context.Context, j tarImportJob, r io.Reader) error { return err } - digestHex := hex.EncodeToString(hasher.Sum(nil)) + digestHex := blobHex(hex.EncodeToString(hasher.Sum(nil))) j.result.digest = images.NewDigest(digestHex) if utils.ValidFile(j.conf.BlobPath(digestHex)) { diff --git a/images/oci/process.go b/images/oci/process.go index fa7027a2..5b80a9f2 100644 --- a/images/oci/process.go +++ b/images/oci/process.go @@ -26,6 +26,7 @@ type layerJob struct { conf *Config idx, total int layer v1.Layer + digestHex string workDir string knownBootHexes map[string]struct{} tracker progress.Tracker @@ -35,11 +36,7 @@ type layerJob struct { func processLayer(ctx context.Context, j layerJob) error { logger := log.WithFunc("oci.processLayer") - layerDigest, err := j.layer.Digest() - if err != nil { - return fmt.Errorf("get digest: %w", err) - } - digestHex := layerDigest.Hex + digestHex := j.digestHex j.result.index = j.idx j.result.digest = images.NewDigest(digestHex) @@ -52,7 +49,7 @@ func processLayer(ctx context.Context, j layerJob) error { logger.Debugf(ctx, "Layer %d: sha256:%s -> erofs (single-pass)", j.idx, digestHex[:12]) // Hoisted out of the retried region: a too-old mkfs.erofs is permanent — retrying burns attempts, backoff sleeps, and aborted blob GETs. - if err = checkErofsVersion(ctx); err != nil { + if err := checkErofsVersion(ctx); err != nil { return err } @@ -61,7 +58,7 @@ func processLayer(ctx context.Context, j layerJob) error { layerUUID := utils.UUIDv5(digestHex) var kernelPath, initrdPath string - err = retryLayer(ctx, pullLayerAttempts, pullLayerBackoff, func() error { + err := retryLayer(ctx, pullLayerAttempts, pullLayerBackoff, func() error { var convErr error kernelPath, initrdPath, convErr = convertLayer(ctx, j, layerDir, digestHex, layerUUID, erofsPath) if convErr != nil { diff --git a/images/oci/pull.go b/images/oci/pull.go index b852886f..ee2c7297 100644 --- a/images/oci/pull.go +++ b/images/oci/pull.go @@ -28,12 +28,16 @@ func pull(ctx context.Context, conf *Config, store *images.Store[imageEntry], im if err != nil { return err } + blobHexes, err := layerBlobHexes(layers) + if err != nil { + return err + } // Heavy work (downloads, EROFS conversion, blob renames) runs outside any transaction under finishImport's per-digest locks; the up-to-date pre-check is racy but benign since the whole flow is idempotent per digest. var upToDate bool var knownBootHexes map[string]struct{} if err := store.View(ctx, func(idx *imageIndex) error { - upToDate = isUpToDate(conf, idx, ref, digestHex) + upToDate = isUpToDate(conf, idx, ref, digestHex, blobHexes) knownBootHexes = collectBootHexes(idx) return nil }); err != nil { @@ -55,7 +59,7 @@ func pull(ctx context.Context, conf *Config, store *images.Store[imageEntry], im results, waitErr := utils.Map(ctx, layers, func(ctx context.Context, i int, layer v1.Layer) (pullLayerResult, error) { var r pullLayerResult err := processLayer(ctx, layerJob{ - conf: conf, idx: i, total: len(layers), layer: layer, + conf: conf, idx: i, total: len(layers), layer: layer, digestHex: blobHexes[i], workDir: workDir, knownBootHexes: knownBootHexes, tracker: tracker, result: &r, }) diff --git a/images/oci/pull_test.go b/images/oci/pull_test.go index ad37443a..a3ed0deb 100644 --- a/images/oci/pull_test.go +++ b/images/oci/pull_test.go @@ -85,7 +85,10 @@ func TestMoveBootFileAndIsUpToDate(t *testing.T) { }, } - if !isUpToDate(conf, idx, "demo", digestHex) { + if !isUpToDate(conf, idx, "demo", digestHex, []string{digestHex}) { t.Fatalf("isUpToDate() = false, want true") } + if isUpToDate(conf, idx, "demo", digestHex, []string{blobHex(digestHex)}) { + t.Fatalf("isUpToDate() for an entry keyed by raw layer digests = true, want false") + } } From 432ef3370d5e78cdb90d0ea3d0e2b737365db05f Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 20:34:51 +0800 Subject: [PATCH 2/3] fix(oci): commit keeps an already published blob Two imports of the same tar convert with different filesystem UUIDs, and the second commit's rename replaced the blob the first had published, so a VM created from the first image later opened a different file. Under the digest lock, a valid blob at the destination now wins and the temp blob is left to the work dir cleanup. --- images/oci/commit.go | 2 +- images/oci/commit_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 images/oci/commit_test.go diff --git a/images/oci/commit.go b/images/oci/commit.go index 5d502687..0f6cad45 100644 --- a/images/oci/commit.go +++ b/images/oci/commit.go @@ -65,7 +65,7 @@ func commitBlobs(conf *Config, results []pullLayerResult) error { r := &results[i] layerDigestHex := r.digest.Hex() - if r.erofsPath != conf.BlobPath(layerDigestHex) { + if r.erofsPath != conf.BlobPath(layerDigestHex) && !utils.ValidFile(conf.BlobPath(layerDigestHex)) { if err := os.Rename(r.erofsPath, conf.BlobPath(layerDigestHex)); err != nil { return fmt.Errorf("move layer %d erofs: %w", r.index, err) } diff --git a/images/oci/commit_test.go b/images/oci/commit_test.go new file mode 100644 index 00000000..9a715455 --- /dev/null +++ b/images/oci/commit_test.go @@ -0,0 +1,36 @@ +package oci + +import ( + "os" + "path/filepath" + "testing" + + "github.com/cocoonstack/cocoon/images" +) + +func TestCommitBlobsKeepsPublishedBlob(t *testing.T) { + root := t.TempDir() + conf := NewConfig(root, 1) + if err := conf.EnsureDirs(); err != nil { + t.Fatalf("EnsureDirs(): %v", err) + } + digestHex := blobHex("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + + for _, content := range []string{"first", "second"} { + tmp := filepath.Join(root, content+".erofs") + if err := os.WriteFile(tmp, []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile(%s): %v", tmp, err) + } + if err := commitBlobs(conf, []pullLayerResult{{digest: images.NewDigest(digestHex), erofsPath: tmp}}); err != nil { + t.Fatalf("commitBlobs(%s): %v", content, err) + } + } + + got, err := os.ReadFile(conf.BlobPath(digestHex)) + if err != nil { + t.Fatalf("ReadFile(blob): %v", err) + } + if string(got) != "first" { + t.Fatalf("published blob = %q, want %q", got, "first") + } +} From db0069a418257142143d39708f7d35e450a494d8 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sat, 17 Oct 2026 20:50:15 +0800 Subject: [PATCH 3/3] fix(os-image): start.sh converts whiteouts; pin the blob key derivation start.sh ran the same mkfs.erofs --tar=f conversion without --aufs, so the local harness kept the whiteout bug; Ubuntu 24.04's erofs-utils 1.7.1 already supports --aufs. TestBlobHex checked sha256 properties instead of the derivation, so a silent re-key of every blob passed; it now pins a golden value. --- images/oci/erofs_test.go | 16 +++------------- os-image/start.sh | 4 ++-- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/images/oci/erofs_test.go b/images/oci/erofs_test.go index 5b22a205..ea533d18 100644 --- a/images/oci/erofs_test.go +++ b/images/oci/erofs_test.go @@ -40,19 +40,9 @@ func TestErofsVersionAtLeast(t *testing.T) { } func TestBlobHex(t *testing.T) { - layerHex := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" - got := blobHex(layerHex) - if got == layerHex { - t.Fatalf("blobHex(%s) reuses the layer digest as the blob key", layerHex) - } - if len(got) != len(layerHex) || strings.Trim(got, "0123456789abcdef") != "" { - t.Fatalf("blobHex(%s) = %q, want 64 lowercase hex chars", layerHex, got) - } - if again := blobHex(layerHex); again != got { - t.Fatalf("blobHex not deterministic: %s != %s", again, got) - } - if other := blobHex(strings.Repeat("f", 64)); other == got { - t.Fatalf("blobHex collides for distinct layers: %s", other) + const want = "b1910ad92b38c2711e188378a0c3e41bc91a32e857795511a1d06b2a1581375c" + if got := blobHex("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); got != want { + t.Fatalf("blobHex() = %s, want %s", got, want) } } diff --git a/os-image/start.sh b/os-image/start.sh index d18f59a3..88e1294d 100644 --- a/os-image/start.sh +++ b/os-image/start.sh @@ -70,9 +70,9 @@ if [ -n "$IMAGE_NAME" ]; then # content digest). Same OCI layer -> same UUID. Pure bash + sha1sum, no Python. LAYER_UUID=$(uuid_v5 "$L") if gzip -t "image_temp/$L" 2>/dev/null; then - gzip -dc "image_temp/$L" | mkfs.erofs --tar=f -zlz4hc -C16384 -T0 -U "$LAYER_UUID" "layer${IDX}.erofs" + gzip -dc "image_temp/$L" | mkfs.erofs --tar=f --aufs -zlz4hc -C16384 -T0 -U "$LAYER_UUID" "layer${IDX}.erofs" else - mkfs.erofs --tar=f -zlz4hc -C16384 -T0 -U "$LAYER_UUID" "layer${IDX}.erofs" "image_temp/$L" + mkfs.erofs --tar=f --aufs -zlz4hc -C16384 -T0 -U "$LAYER_UUID" "layer${IDX}.erofs" "image_temp/$L" fi IDX=$((IDX+1)) done