From 78f96396dfebe0aa60a32c647b763729d5e0b6a8 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Mon, 21 Sep 2026 11:17:24 +0200 Subject: [PATCH 1/2] + executing coverage for the ReFS offline block clone RefsOfflineBlockCloner and RefsCowBlockRefcountEditor had no test that ever ran. Their only caller was RefsBlockCloneExternalTests, which sits in the ExternalFsInterop tier the core CI filter excludes, needs CWB_REFS_CLONE_IMAGE and CWB_REFS_FOREFST that nothing sets, generates no fixture, and runs under continue-on-error - so it always reached Assert.Ignore. A 263-line primitive whose failure mode is silent cross-file extent corruption was therefore shipped on the strength of a placeholder. Root cause of the gap is structural, not neglect: this package deliberately has no ReFS creator - the writer edits existing volumes only - so there was no image to drive the cloner against, and nobody wrote one. RefsSyntheticVolume now builds a byte-exact unmounted ReFS 3.14 image: VBR, SUPB, two CHKP slots, Container Table, Object Table, Schema Table, Medium Allocator, Block Refcount and a root-directory object, every tree a single leaf page so no B+ split, separator-key convention or Upcase Table is involved. RefsImageProbe decodes the result straight from raw bytes and shares no code with the reader, writer or CoW engine, so the assertions are not our reader agreeing with our writer. Covered, running in the core tier: whole-file clone of two equal-sized cluster-aligned extent-backed files; first-clone materialisation both of an absent Block Refcount row and of a zero slot in an existing one; paired publication of Object Table root #0 and Block Refcount root #6 through the alternate checkpoint with the superseded slot left intact; allocator roots accounting for the newly reserved CoW pages; reclamation of the destination's former allocation only after the checkpoint is durable; an abort injected into the checkpoint write leaving the volume byte-identical; copy-on-write isolation when one clone is written afterwards; fragmented sources carrying every run; and refusal of unequal sizes, non-aligned sizes, resident streams, empty streams, repeated sharing, self-clone and unknown destinations. Every assertion was verified to bite by perturbing the code it covers: removing the implicit-owner baseline from AddCloneReferences, skipping the sparse-row insertion, publishing root #0 without root #6, reclaiming before the commit, and an off-by-one on the cloned extent each fail the tests that name them. The cloner itself turned out to be correct on its stated profile. --- Compression.Tests/Refs/RefsBlockCloneTests.cs | 471 +++++++++++++++ Compression.Tests/Refs/RefsImageProbe.cs | 390 +++++++++++++ Compression.Tests/Refs/RefsSyntheticVolume.cs | 549 ++++++++++++++++++ Hawkynt.FileFormats.FileSystems/README.md | 2 +- 4 files changed, 1411 insertions(+), 1 deletion(-) create mode 100644 Compression.Tests/Refs/RefsBlockCloneTests.cs create mode 100644 Compression.Tests/Refs/RefsImageProbe.cs create mode 100644 Compression.Tests/Refs/RefsSyntheticVolume.cs diff --git a/Compression.Tests/Refs/RefsBlockCloneTests.cs b/Compression.Tests/Refs/RefsBlockCloneTests.cs new file mode 100644 index 000000000..522986a3a --- /dev/null +++ b/Compression.Tests/Refs/RefsBlockCloneTests.cs @@ -0,0 +1,471 @@ +#pragma warning disable CS1591 +using Compression.Registry; +using FileSystem.Refs; + +namespace Compression.Tests.Refs; + +/// +/// Executing coverage for the offline ReFS whole-file block clone. +/// +/// and +/// previously had no test that ran: their only caller was an external-oracle +/// fixture that skips unless two environment variables point at a real ReFS +/// image, which CI never sets. A wrong block clone corrupts across files +/// silently — two streams sharing clusters they should not, or reference counts +/// that disagree with reality — so the primitive is driven here against images +/// built by and checked with +/// , which decodes the resulting bytes without going +/// back through the code that wrote them. +/// +[TestFixture] +public sealed class RefsBlockCloneTests { + private const int ClusterSize = RefsSyntheticVolume.ClusterSize; + + private static byte[] Pattern(byte seed, int length) { + var result = new byte[length]; + for (var i = 0; i < length; ++i) result[i] = (byte)(seed + i * 31); + return result; + } + + [Test, Category("HappyPath")] + public void SyntheticVolume_IsReadableAsARefsNamespace() { + var source = Pattern(0x11, 2 * ClusterSize); + var destination = Pattern(0x77, 2 * ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", source) + .WithFile("bravo.bin", destination) + .Build(); + + using var stream = new MemoryStream(image, writable: true); + var metadata = RefsMetadataReader.Open(stream); + var files = new RefsNamespaceReader(metadata).ReadAll(); + + Assert.Multiple(() => { + Assert.That(files.Select(f => f.Path), Is.EquivalentTo(new[] { "alpha.bin", "bravo.bin" })); + var alpha = files.Single(f => f.Path == "alpha.bin"); + Assert.That(alpha.IsResident, Is.False); + Assert.That(alpha.Size, Is.EqualTo(source.Length)); + Assert.That(alpha.AllocatedSize, Is.EqualTo(source.Length)); + Assert.That(alpha.Extents, Has.Count.EqualTo(1)); + Assert.That(alpha.Extents[0].ClusterCount, Is.EqualTo(2)); + }); + + var probe = new RefsImageProbe(image); + var probed = probe.ReadFiles(probe.ActiveCheckpoint()); + Assert.Multiple(() => { + Assert.That(probe.ReadFileContent(probed["alpha.bin"]), Is.EqualTo(source)); + Assert.That(probe.ReadFileContent(probed["bravo.bin"]), Is.EqualTo(destination)); + }); + } + + [Test, Category("HappyPath")] + public void WholeFileClone_SharesTheSourceClustersAndMaterialisesCountTwo() { + var sourceBytes = Pattern(0x11, 2 * ClusterSize); + var destinationBytes = Pattern(0x77, 2 * ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", sourceBytes) + .WithFile("bravo.bin", destinationBytes) + .Build(); + + var before = new RefsImageProbe(image); + var beforeCheckpoint = before.ActiveCheckpoint(); + var beforeFiles = before.ReadFiles(beforeCheckpoint); + var sourceClusters = beforeFiles["alpha.bin"].Clusters; + var formerDestinationClusters = beforeFiles["bravo.bin"].Clusters; + Assume.That(before.CountRefcountRows(beforeCheckpoint), Is.Zero, + "the fixture must start with a sparse Block Refcount tree"); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var afterFiles = after.ReadFiles(afterCheckpoint); + var refcounts = after.ReadRefcounts(afterCheckpoint); + var allocated = after.ReadAllocatedClusters(afterCheckpoint); + + Assert.Multiple(() => { + // Structural: both stream layouts now name the same physical clusters. + Assert.That(afterFiles["alpha.bin"].Clusters, Is.EqualTo(sourceClusters)); + Assert.That(afterFiles["bravo.bin"].Clusters, Is.EqualTo(sourceClusters)); + Assert.That(afterFiles["bravo.bin"].Size, Is.EqualTo((ulong)sourceBytes.Length)); + Assert.That(afterFiles["bravo.bin"].AllocatedSize, Is.EqualTo((ulong)sourceBytes.Length)); + + // Structural: every shared cluster carries reference count 2. + foreach (var cluster in sourceClusters) { + Assert.That(refcounts.ContainsKey(cluster), Is.True, + $"PLCN 0x{cluster:X} has no Block Refcount row after the clone."); + Assert.That(refcounts[cluster].Count, Is.EqualTo(2), + $"PLCN 0x{cluster:X} is not shared by exactly two streams."); + } + + // Structural: the row's own total field agrees with its words. + foreach (var (start, totals) in after.ReadRefcountTotals(afterCheckpoint)) + Assert.That(totals.Stated, Is.EqualTo(totals.Actual), + $"Block Refcount row 0x{start:X} states a total its words do not add up to."); + + // Round-trip: both names read back as the source content. + Assert.That(after.ReadFileContent(afterFiles["alpha.bin"]), Is.EqualTo(sourceBytes)); + Assert.That(after.ReadFileContent(afterFiles["bravo.bin"]), Is.EqualTo(sourceBytes)); + + // Structural: the destination's former allocation went back to the pool. + foreach (var cluster in formerDestinationClusters) + Assert.That(allocated, Does.Not.Contain(cluster), + $"The destination's former PLCN 0x{cluster:X} is still marked allocated."); + foreach (var cluster in sourceClusters) + Assert.That(allocated, Does.Contain(cluster), + $"Shared PLCN 0x{cluster:X} must stay allocated."); + }); + } + + [Test, Category("HappyPath")] + public void FirstClone_OnAZeroSlotOfAnExistingRow_BecomesTwoWithoutAddingARow() { + var sourceBytes = Pattern(0x21, ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", sourceBytes) + .WithFile("bravo.bin", Pattern(0x5A, ClusterSize)) + .WithEmptyRefcountRow() + .Build(); + + var before = new RefsImageProbe(image); + var beforeCheckpoint = before.ActiveCheckpoint(); + var sourceClusters = before.ReadFiles(beforeCheckpoint)["alpha.bin"].Clusters; + Assert.Multiple(() => { + Assert.That(before.CountRefcountRows(beforeCheckpoint), Is.EqualTo(1), + "the fixture must start with exactly one all-zero Block Refcount row"); + Assert.That(before.ReadRefcounts(beforeCheckpoint), Is.Empty, + "no slot of the seeded row may carry a non-zero word yet"); + }); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var refcounts = after.ReadRefcounts(afterCheckpoint); + + Assert.Multiple(() => { + Assert.That(after.CountRefcountRows(afterCheckpoint), Is.EqualTo(1), + "an existing row must be updated, not duplicated"); + foreach (var cluster in sourceClusters) + Assert.That(refcounts[cluster].Count, Is.EqualTo(2), + $"A zero slot must become 2 on first sharing, not 1 — PLCN 0x{cluster:X}."); + Assert.That(refcounts, Has.Count.EqualTo(sourceClusters.Count), + "only the shared clusters may gain a non-zero reference word"); + foreach (var (start, totals) in after.ReadRefcountTotals(afterCheckpoint)) + Assert.That(totals.Stated, Is.EqualTo(totals.Actual), + $"Block Refcount row 0x{start:X} states a stale total."); + }); + } + + [Test, Category("HappyPath")] + public void Clone_PublishesObjectTableAndRefcountRootsInOneAlternateCheckpoint() { + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", Pattern(0x31, 2 * ClusterSize)) + .WithFile("bravo.bin", Pattern(0x8F, 2 * ClusterSize)) + .Build(); + + var before = new RefsImageProbe(image); + var beforeCheckpoint = before.ActiveCheckpoint(); + var beforeFiles = before.ReadFiles(beforeCheckpoint); + var formerDestinationClusters = beforeFiles["bravo.bin"].Clusters; + var beforeAllocated = before.ReadAllocatedClusters(beforeCheckpoint); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var stale = after.ReadCheckpoint(beforeCheckpoint.Lcn); + var afterAllocated = after.ReadAllocatedClusters(afterCheckpoint); + + Assert.Multiple(() => { + // The publication is an alternate-slot swap, not an in-place edit. + Assert.That(afterCheckpoint.Lcn, Is.Not.EqualTo(beforeCheckpoint.Lcn)); + Assert.That(afterCheckpoint.Clock, Is.GreaterThan(beforeCheckpoint.Clock)); + + // Object Table #0 and Block Refcount #6 moved in the same publication; + // a checkpoint carrying one without the other is the silent-corruption + // case this primitive exists to avoid. + Assert.That(afterCheckpoint.Roots[0], Is.Not.EqualTo(beforeCheckpoint.Roots[0]), + "Object Table root #0 did not move."); + Assert.That(afterCheckpoint.Roots[6], Is.Not.EqualTo(beforeCheckpoint.Roots[6]), + "Block Refcount root #6 did not move."); + Assert.That(afterCheckpoint.Roots[1], Is.Not.EqualTo(beforeCheckpoint.Roots[1]), + "Medium Allocator root #1 must account for the newly reserved CoW pages."); + Assert.That(afterCheckpoint.Roots[7], Is.EqualTo(beforeCheckpoint.Roots[7]), + "The Container Table has no business changing during a block clone."); + + // The superseded slot is untouched and still describes the pre-clone volume. + Assert.That(stale.Clock, Is.EqualTo(beforeCheckpoint.Clock)); + Assert.That(stale.Roots[0], Is.EqualTo(beforeCheckpoint.Roots[0])); + Assert.That(stale.Roots[6], Is.EqualTo(beforeCheckpoint.Roots[6])); + Assert.That(after.ReadFiles(stale)["bravo.bin"].Clusters, Is.EqualTo(formerDestinationClusters)); + Assert.That(after.CountRefcountRows(stale), Is.Zero); + + // Every page the replacement roots point at is newly reserved storage that + // the published allocator now owns. + foreach (var rootIndex in new[] { 0, 1, 6 }) + foreach (var lcn in afterCheckpoint.Roots[rootIndex]) { + var physical = after.ToPhysical(lcn); + Assert.That(beforeAllocated, Does.Not.Contain(physical), + $"Root #{rootIndex} was rebuilt over live PLCN 0x{physical:X}."); + Assert.That(afterAllocated, Does.Contain(physical), + $"Root #{rootIndex} page PLCN 0x{physical:X} is not marked allocated by the published allocator."); + } + + // Reclamation is visible only through the new checkpoint. + foreach (var cluster in formerDestinationClusters) { + Assert.That(beforeAllocated, Does.Contain(cluster)); + Assert.That(afterAllocated, Does.Not.Contain(cluster)); + } + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_AbortedWhilePublishing_LeavesTheVolumeExactlyAsItWas() { + var sourceBytes = Pattern(0x41, 2 * ClusterSize); + var destinationBytes = Pattern(0x9D, 2 * ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", sourceBytes) + .WithFile("bravo.bin", destinationBytes) + .Build(); + + var before = new RefsImageProbe(image); + var beforeCheckpoint = before.ActiveCheckpoint(); + var beforeFiles = before.ReadFiles(beforeCheckpoint); + var beforeAllocated = before.ReadAllocatedClusters(beforeCheckpoint); + var alternate = before.CheckpointLcns.Single(lcn => lcn != beforeCheckpoint.Lcn); + + using (var stream = new WriteBarrierStream( + image, + (long)alternate * ClusterSize, + (long)(alternate + (ulong)before.ClustersPerPage) * ClusterSize)) { + Assert.Throws(() => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin")); + Assert.That(stream.Tripped, Is.True, "the abort must happen while the alternate checkpoint is written"); + } + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var afterFiles = after.ReadFiles(afterCheckpoint); + + Assert.Multiple(() => { + Assert.That(afterCheckpoint.Lcn, Is.EqualTo(beforeCheckpoint.Lcn), + "a failed publication must not change which checkpoint is active"); + Assert.That(afterCheckpoint.Clock, Is.EqualTo(beforeCheckpoint.Clock)); + Assert.That(afterCheckpoint.Roots, Is.EqualTo(beforeCheckpoint.Roots)); + Assert.That(after.CountRefcountRows(afterCheckpoint), Is.Zero, + "no Block Refcount row may become live through an aborted clone"); + Assert.That(afterFiles["alpha.bin"].Clusters, Is.EqualTo(beforeFiles["alpha.bin"].Clusters)); + Assert.That(afterFiles["bravo.bin"].Clusters, Is.EqualTo(beforeFiles["bravo.bin"].Clusters)); + Assert.That(after.ReadFileContent(afterFiles["alpha.bin"]), Is.EqualTo(sourceBytes)); + Assert.That(after.ReadFileContent(afterFiles["bravo.bin"]), Is.EqualTo(destinationBytes)); + Assert.That(after.ReadAllocatedClusters(afterCheckpoint), Is.EquivalentTo(beforeAllocated), + "the destination's allocation must not be reclaimed before the clone is durable"); + }); + } + + [Test, Category("HappyPath")] + public void WritingToOneCloneAfterwards_LeavesTheOtherUntouched() { + var sourceBytes = Pattern(0x51, 2 * ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", sourceBytes) + .WithFile("bravo.bin", Pattern(0xB3, 2 * ClusterSize)) + .Build(); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var cloned = new RefsImageProbe(image); + var shared = cloned.ReadFiles(cloned.ActiveCheckpoint())["alpha.bin"].Clusters; + var replacement = Pattern(0xE7, 2 * ClusterSize); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineModifier.Add(stream, [ArchiveInputInfo.InMemory("bravo.bin", replacement)]); + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var afterFiles = after.ReadFiles(afterCheckpoint); + var refcounts = after.ReadRefcounts(afterCheckpoint); + var allocated = after.ReadAllocatedClusters(afterCheckpoint); + + Assert.Multiple(() => { + Assert.That(afterFiles["alpha.bin"].Clusters, Is.EqualTo(shared), + "the untouched clone must keep its extents"); + Assert.That(after.ReadFileContent(afterFiles["alpha.bin"]), Is.EqualTo(sourceBytes), + "writing to one clone changed the other's bytes"); + Assert.That(afterFiles["bravo.bin"].Clusters, Is.Not.EqualTo(shared), + "the written clone must land on fresh clusters, not over the shared ones"); + Assert.That(after.ReadFileContent(afterFiles["bravo.bin"]), Is.EqualTo(replacement)); + + foreach (var cluster in shared) { + Assert.That(allocated, Does.Contain(cluster), + $"Shared PLCN 0x{cluster:X} was freed while a stream still references it."); + var count = refcounts.TryGetValue(cluster, out var word) ? word.Count : (ushort)0; + Assert.That(count, Is.LessThanOrEqualTo(1), + $"PLCN 0x{cluster:X} still claims more than one owner after the second stream moved away."); + } + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesUnequalLogicalSizes() { + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", Pattern(0x61, ClusterSize)) + .WithFile("bravo.bin", Pattern(0xC1, 2 * ClusterSize)) + .Build(); + var untouched = (byte[])image.Clone(); + + using var stream = new MemoryStream(image, writable: true); + var error = Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin")); + Assert.Multiple(() => { + Assert.That(error!.Message, Does.Contain("equal source/destination logical sizes")); + Assert.That(image, Is.EqualTo(untouched), "a refused clone must not touch a single byte"); + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesNonClusterAlignedFiles() { + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", Pattern(0x71, ClusterSize - 17)) + .WithFile("bravo.bin", Pattern(0xD1, ClusterSize - 17)) + .Build(); + var untouched = (byte[])image.Clone(); + + using var stream = new MemoryStream(image, writable: true); + var error = Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin")); + Assert.Multiple(() => { + Assert.That(error!.Message, Does.Contain("cluster-aligned logical size")); + Assert.That(image, Is.EqualTo(untouched)); + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesResidentStreams() { + var content = Pattern(0x81, 48); + var image = new RefsSyntheticVolume() + .WithResidentFile("alpha.bin", content) + .WithFile("bravo.bin", Pattern(0xE1, ClusterSize)) + .Build(); + var untouched = (byte[])image.Clone(); + + using var stream = new MemoryStream(image, writable: true); + var error = Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin")); + Assert.Multiple(() => { + Assert.That(error!.Message, Does.Contain("extent-backed")); + Assert.That(image, Is.EqualTo(untouched)); + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesEmptyStreams() { + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", []) + .WithFile("bravo.bin", []) + .Build(); + var untouched = (byte[])image.Clone(); + + using var stream = new MemoryStream(image, writable: true); + Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin")); + Assert.That(image, Is.EqualTo(untouched)); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesAlreadySharedSourceClusters() { + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", Pattern(0x91, ClusterSize)) + .WithFile("bravo.bin", Pattern(0xA1, ClusterSize)) + .WithFile("carol.bin", Pattern(0xB1, ClusterSize)) + .Build(); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var afterFirst = (byte[])image.Clone(); + using var second = new MemoryStream(image, writable: true); + var error = Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(second, "alpha.bin", "carol.bin")); + Assert.Multiple(() => { + Assert.That(error!.Message, Does.Contain("already shared")); + Assert.That(image, Is.EqualTo(afterFirst), + "a refused repeat clone must not half-apply over the first one"); + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesTheSameFileAsSourceAndDestination() { + var image = new RefsSyntheticVolume().WithFile("alpha.bin", Pattern(0xA5, ClusterSize)).Build(); + using var stream = new MemoryStream(image, writable: true); + Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "alpha.bin")); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesAnUnknownDestination() { + var image = new RefsSyntheticVolume().WithFile("alpha.bin", Pattern(0xA6, ClusterSize)).Build(); + using var stream = new MemoryStream(image, writable: true); + Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "ghost.bin")); + } + + [Test, Category("EdgeCase")] + public void Clone_OfAFragmentedSourceCarriesEveryRunToTheDestination() { + var sourceBytes = Pattern(0xB7, 4 * ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", sourceBytes, fragments: 3) + .WithFile("bravo.bin", Pattern(0xC7, 4 * ClusterSize)) + .Build(); + + var before = new RefsImageProbe(image); + var sourceView = before.ReadFiles(before.ActiveCheckpoint())["alpha.bin"]; + Assume.That(sourceView.Extents, Has.Count.EqualTo(3), "the fixture must fragment the source"); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var afterFiles = after.ReadFiles(afterCheckpoint); + var refcounts = after.ReadRefcounts(afterCheckpoint); + + Assert.Multiple(() => { + Assert.That(afterFiles["bravo.bin"].Extents, Has.Count.EqualTo(3), + "a fragmented clone must reproduce every run, not collapse or drop one"); + Assert.That(afterFiles["bravo.bin"].Clusters, Is.EqualTo(sourceView.Clusters)); + Assert.That(afterFiles["bravo.bin"].Extents.Select(e => (e.Vcn, e.Lcn, e.Run)), + Is.EqualTo(sourceView.Extents.Select(e => (e.Vcn, e.Lcn, e.Run)))); + foreach (var cluster in sourceView.Clusters) + Assert.That(refcounts[cluster].Count, Is.EqualTo(2)); + Assert.That(after.ReadFileContent(afterFiles["bravo.bin"]), Is.EqualTo(sourceBytes)); + Assert.That(after.ReadFileContent(afterFiles["alpha.bin"]), Is.EqualTo(sourceBytes)); + }); + } + + /// A memory image whose writes fail inside one byte range, to + /// simulate losing the medium at the exact moment of publication. + private sealed class WriteBarrierStream(byte[] buffer, long failFrom, long failTo) + : MemoryStream(buffer, writable: true) { + public bool Tripped { get; private set; } + + public override void Write(byte[] buffer, int offset, int count) { + this.Guard(count); + base.Write(buffer, offset, count); + } + + public override void Write(ReadOnlySpan buffer) { + this.Guard(buffer.Length); + base.Write(buffer); + } + + private void Guard(int count) { + var start = this.Position; + if (start + count <= failFrom || start >= failTo) return; + this.Tripped = true; + throw new IOException("Simulated media failure while publishing the alternate ReFS checkpoint."); + } + } +} diff --git a/Compression.Tests/Refs/RefsImageProbe.cs b/Compression.Tests/Refs/RefsImageProbe.cs new file mode 100644 index 000000000..d4066dd16 --- /dev/null +++ b/Compression.Tests/Refs/RefsImageProbe.cs @@ -0,0 +1,390 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; + +namespace Compression.Tests.Refs; + +/// +/// Independent on-disk decoder for ReFS images, written straight from the +/// structure definitions and deliberately sharing no code with the package's +/// reader, writer or CoW engine. +/// +/// Its whole purpose is to be a second pair of eyes: assertions that go through +/// RefsMetadataReader only prove that our reader agrees with our writer, +/// which is exactly how a shared defect survives. Everything this type returns +/// is decoded from raw bytes — checkpoint clocks, root descriptors, B+ rows, +/// Block Refcount words, allocator bitmaps and extent tables. +/// +internal sealed class RefsImageProbe { + public const int SuperblockCluster = 0x1E; + + private readonly byte[] _image; + + public RefsImageProbe(byte[] image) { + this._image = image; + if (!image.AsSpan(3, 4).SequenceEqual("ReFS"u8)) throw new InvalidDataException("Not a ReFS image."); + var sectorSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(0x20, 4)); + var sectorsPerCluster = (int)BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(0x24, 4)); + this.ClusterSize = sectorSize * sectorsPerCluster; + this.PageSize = this.ClusterSize <= 4096 ? 16 * 1024 : 64 * 1024; + this.ClustersPerPage = this.PageSize / this.ClusterSize; + this.LoadBootstrap(); + this.LoadContainers(); + } + + public int ClusterSize { get; } + public int PageSize { get; } + public int ClustersPerPage { get; } + public int PageReferenceSize { get; private set; } + public IReadOnlyList CheckpointLcns { get; private set; } = []; + public ulong ActiveCheckpointLcn { get; private set; } + public ulong ActiveCheckpointClock { get; private set; } + + private readonly Dictionary _containerBases = []; + private uint _clustersPerContainer; + private int _containerShift; + + // ── bootstrap ────────────────────────────────────────────────────────────── + + private void LoadBootstrap() { + var supb = this.Cluster(SuperblockCluster); + if (!supb.AsSpan(0, 4).SequenceEqual("SUPB"u8)) + throw new InvalidDataException("Probe found no SUPB at cluster 0x1E."); + var listOffset = (int)BinaryPrimitives.ReadUInt32LittleEndian(supb.AsSpan(0x70, 4)); + var listCount = (int)BinaryPrimitives.ReadUInt32LittleEndian(supb.AsSpan(0x74, 4)); + var slots = new List(listCount); + for (var i = 0; i < listCount; ++i) { + var lcn = BinaryPrimitives.ReadUInt64LittleEndian(supb.AsSpan(listOffset + i * 8, 8)); + if (lcn is not (0 or ulong.MaxValue)) slots.Add(lcn); + } + this.CheckpointLcns = slots; + + ulong bestLcn = 0; + ulong bestClock = 0; + var found = false; + foreach (var lcn in slots) { + var page = this.Page(lcn); + if (page == null || !page.AsSpan(0, 4).SequenceEqual("CHKP"u8)) continue; + var clock = BinaryPrimitives.ReadUInt64LittleEndian(page.AsSpan(0x60, 8)); + if (found && clock <= bestClock) continue; + bestLcn = lcn; + bestClock = clock; + found = true; + } + if (!found) throw new InvalidDataException("Probe found no readable CHKP."); + this.ActiveCheckpointLcn = bestLcn; + this.ActiveCheckpointClock = bestClock; + this.PageReferenceSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(this.Page(bestLcn)!.AsSpan(0x5C, 4)); + } + + /// Clock and the thirteen root page addresses of one checkpoint slot. + internal sealed record CheckpointView(ulong Lcn, ulong Clock, IReadOnlyList> Roots); + + public CheckpointView ReadCheckpoint(ulong lcn) { + var page = this.Page(lcn) ?? throw new InvalidDataException($"Checkpoint 0x{lcn:X} is unreadable."); + if (!page.AsSpan(0, 4).SequenceEqual("CHKP"u8)) + throw new InvalidDataException($"Cluster 0x{lcn:X} does not contain CHKP."); + var clock = BinaryPrimitives.ReadUInt64LittleEndian(page.AsSpan(0x60, 8)); + var referenceSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(0x5C, 4)); + var flags = BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(0x78, 4)); + var listBase = (flags & 0x0200) != 0 + ? (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(0x94, 4)) + : 0x94; + + var roots = new List>(13); + for (var i = 0; i < 13; ++i) { + var descriptor = (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(listBase + i * 4, 4)); + if (descriptor <= 0 || descriptor + referenceSize > page.Length) { + roots.Add([]); + continue; + } + roots.Add(ParseReference(page.AsSpan(descriptor, referenceSize))); + } + return new CheckpointView(lcn, clock, roots); + } + + public CheckpointView ActiveCheckpoint() => this.ReadCheckpoint(this.ActiveCheckpointLcn); + + private static IReadOnlyList ParseReference(ReadOnlySpan reference) { + var lcns = new List(4); + for (var i = 0; i < 4 && i * 8 + 8 <= reference.Length; ++i) { + var lcn = BinaryPrimitives.ReadUInt64LittleEndian(reference.Slice(i * 8, 8)); + if (lcn is not (0 or ulong.MaxValue)) lcns.Add(lcn); + } + return lcns; + } + + // ── container translation ────────────────────────────────────────────────── + + private void LoadContainers() { + var active = this.ActiveCheckpoint(); + foreach (var rootIndex in new[] { 7, 8 }) { + var root = active.Roots[rootIndex]; + if (root.Count == 0) continue; + var bases = new Dictionary(); + uint cpc = 0; + foreach (var (key, value) in this.WalkPhysicalTree(root)) { + if (key.Length < 16 || value.Length < 0x98) continue; + var id = BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(0, 8)); + var rowCpc = BinaryPrimitives.ReadUInt32LittleEndian(value.AsSpan(0x18, 4)); + if (rowCpc == 0) continue; + if (cpc == 0) cpc = rowCpc; + if (rowCpc != cpc) { bases.Clear(); break; } + bases[id] = BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(value.Length - 16, 8)); + } + if (bases.Count == 0 || cpc == 0 || (cpc & (cpc - 1)) != 0) continue; + foreach (var item in bases) this._containerBases[item.Key] = item.Value; + this._clustersPerContainer = cpc; + this._containerShift = 0; + while ((1u << this._containerShift) < cpc) ++this._containerShift; + return; + } + throw new InvalidDataException("Probe could not decode a Container Table."); + } + + public ulong ToPhysical(ulong virtualLcn) { + var container = virtualLcn >> this._containerShift; + var within = virtualLcn & (this._clustersPerContainer - 1UL); + if (!this._containerBases.TryGetValue(container, out var physicalBase)) + throw new InvalidDataException($"Probe cannot map virtual container {container}."); + return physicalBase + within; + } + + // ── B+ walking ───────────────────────────────────────────────────────────── + + public IReadOnlyList<(byte[] Key, byte[] Value)> WalkVirtualTree(IReadOnlyList rootLcns) + => this.Walk(rootLcns.Select(this.ToPhysical).ToArray(), virtualChildren: true); + + public IReadOnlyList<(byte[] Key, byte[] Value)> WalkPhysicalTree(IReadOnlyList rootLcns) + => this.Walk(rootLcns, virtualChildren: false); + + private IReadOnlyList<(byte[] Key, byte[] Value)> Walk( + IReadOnlyList physicalLcns, + bool virtualChildren) { + var result = new List<(byte[], byte[])>(); + var seen = new HashSet(); + var pending = new Stack>(); + pending.Push(physicalLcns); + + while (pending.Count > 0) { + var slots = pending.Pop(); + if (slots.Count == 0 || !seen.Add(slots[0])) continue; + var page = this.PageFromSlots(slots); + if (page == null || !page.AsSpan(0, 4).SequenceEqual("MSB+"u8)) continue; + + var nodeOffset = 0x50 + (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(0x50, 4)); + var isInner = (BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(nodeOffset + 0x0C, 4)) & 0x100) != 0; + var indexStart = nodeOffset + (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(nodeOffset + 0x10, 4)); + var indexEnd = nodeOffset + (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(nodeOffset + 0x20, 4)); + + for (var entry = indexStart; entry < indexEnd; entry += 4) { + var rowOffset = nodeOffset + (int)(BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(entry, 4)) & 0xFFFF); + var rowSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(rowOffset, 4)); + if (rowSize < 16 || rowOffset + rowSize > page.Length) continue; + var keyOffset = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(rowOffset + 4, 2)); + var keyLength = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(rowOffset + 6, 2)); + var valueOffset = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(rowOffset + 10, 2)); + var valueLength = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(rowOffset + 12, 2)); + if (keyOffset + keyLength > rowSize || valueOffset + valueLength > rowSize) continue; + var key = page.AsSpan(rowOffset + keyOffset, keyLength).ToArray(); + var value = page.AsSpan(rowOffset + valueOffset, valueLength).ToArray(); + + if (!isInner) { + result.Add((key, value)); + continue; + } + if (value.Length < 32) continue; + var child = ParseReference(value); + if (child.Count == 0) continue; + pending.Push(virtualChildren ? child.Select(this.ToPhysical).ToArray() : child); + } + } + return result; + } + + // ── decoded views ────────────────────────────────────────────────────────── + + /// One decoded Block Refcount word: the 14-bit count and its two flag bits. + internal readonly record struct RefcountWord(ushort Count, bool DedupMetadata, bool DedupManaged); + + /// + /// Decodes every tracked Block Refcount slot of root #6 straight from the + /// 0x820-byte rows. Only slots inside an existing row appear; absence is the + /// sparse "one ordinary owner" state. + /// + public IReadOnlyDictionary ReadRefcounts(CheckpointView checkpoint) { + var result = new Dictionary(); + foreach (var (key, value) in this.WalkVirtualTree(checkpoint.Roots[6])) { + if (key.Length < 16 || value.Length < 0x820) continue; + var start = BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(0, 8)); + if (BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(8, 8)) != 0x400) continue; + for (var i = 0; i < 0x400; ++i) { + var raw = BinaryPrimitives.ReadUInt16LittleEndian(value.AsSpan(0x1C + i * 2, 2)); + if (raw == 0) continue; + result[start + (ulong)i] = new RefcountWord( + (ushort)(raw & 0x3FFF), + (raw & 0x4000) != 0, + (raw & 0x8000) != 0); + } + } + return result; + } + + /// Sum of the low 14-bit counts each Block Refcount row advertises at +0x18. + public IReadOnlyDictionary ReadRefcountTotals(CheckpointView checkpoint) { + var result = new Dictionary(); + foreach (var (key, value) in this.WalkVirtualTree(checkpoint.Roots[6])) { + if (key.Length < 16 || value.Length < 0x820) continue; + if (BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(8, 8)) != 0x400) continue; + var start = BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(0, 8)); + uint actual = 0; + for (var i = 0; i < 0x400; ++i) + actual += (uint)(BinaryPrimitives.ReadUInt16LittleEndian(value.AsSpan(0x1C + i * 2, 2)) & 0x3FFF); + result[start] = (BinaryPrimitives.ReadUInt32LittleEndian(value.AsSpan(0x18, 4)), actual); + } + return result; + } + + public int CountRefcountRows(CheckpointView checkpoint) + => this.WalkVirtualTree(checkpoint.Roots[6]) + .Count(row => row.Key.Length >= 16 + && BinaryPrimitives.ReadUInt64LittleEndian(row.Key.AsSpan(8, 8)) == 0x400); + + /// Allocation state of the Medium Allocator (root #1) as a physical-LCN set. + public HashSet ReadAllocatedClusters(CheckpointView checkpoint) { + var allocated = new HashSet(); + foreach (var (_, value) in this.WalkVirtualTree(checkpoint.Roots[1])) { + if (value.Length < 24) continue; + var start = BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(0x00, 8)); + var length = BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(0x08, 8)); + var flags = BinaryPrimitives.ReadUInt16LittleEndian(value.AsSpan(0x12, 2)); + if (length == 0 || length > 16384) continue; + for (ulong i = 0; i < length; ++i) { + var isAllocated = flags switch { + 0x01 when value.Length >= 0x18 + 2048 => (value[0x18 + (int)(i >> 3)] & (1 << (int)(i & 7))) != 0, + 0x02 => true, + 0x05 or 0x09 => false, + _ => throw new InvalidDataException($"Probe met an allocator row with flags 0x{flags:X4}."), + }; + if (isAllocated) allocated.Add(this.ToPhysical(start + i)); + } + } + return allocated; + } + + /// A file's on-disk stream layout, decoded from its type-0x40 backing row. + internal sealed record FileView( + string Name, + ulong FileId, + ulong Size, + ulong AllocatedSize, + IReadOnlyList<(uint Vcn, ulong Lcn, uint Run, uint Flags)> Extents) { + public IReadOnlyList Clusters { + get { + var result = new List(); + foreach (var extent in this.Extents.OrderBy(e => e.Vcn)) + for (uint i = 0; i < extent.Run; ++i) + result.Add(extent.Lcn + i); + return result; + } + } + } + + public IReadOnlyDictionary ReadFiles(CheckpointView checkpoint) { + var objects = new Dictionary>(); + foreach (var (key, value) in this.WalkVirtualTree(checkpoint.Roots[0])) { + if (key.Length < 16 || value.Length < 0x40) continue; + var oid = BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(8, 8)); + var reference = ParseReference(value.AsSpan(0x20)); + if (reference.Count > 0) objects[oid] = reference; + } + if (!objects.TryGetValue(RefsSyntheticVolume.RootDirectoryOid, out var directoryRoot)) + throw new InvalidDataException("Probe found no root directory object."); + + var rows = this.WalkVirtualTree(directoryRoot); + var backing = new Dictionary(); + foreach (var (key, value) in rows) { + if (key.Length < 16 || BinaryPrimitives.ReadUInt16LittleEndian(key.AsSpan(0, 2)) != 0x40) continue; + backing[BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(8, 8))] = value; + } + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var (key, value) in rows) { + if (key.Length < 4 || BinaryPrimitives.ReadUInt16LittleEndian(key.AsSpan(0, 2)) != 0x30) continue; + if (BinaryPrimitives.ReadUInt16LittleEndian(key.AsSpan(2, 2)) != 0x02) continue; + if (value.Length is < 0x48 or > 0x54) continue; + var name = Encoding.Unicode.GetString(key.AsSpan(4)).TrimEnd('\0'); + var fileId = BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(0x00, 8)); + if (!backing.TryGetValue(fileId, out var holder)) continue; + result[name] = new FileView( + name, + fileId, + BinaryPrimitives.ReadUInt64LittleEndian(holder.AsSpan(0x58, 8)), + BinaryPrimitives.ReadUInt64LittleEndian(holder.AsSpan(0x60, 8)), + this.DecodeExtents(holder)); + } + return result; + } + + private IReadOnlyList<(uint Vcn, ulong Lcn, uint Run, uint Flags)> DecodeExtents(byte[] holder) { + var result = new List<(uint, ulong, uint, uint)>(); + if (holder.Length < 0xB0) return result; + var recordSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(holder.AsSpan(0xA8, 4)); + if (recordSize <= 0 || 0xA8 + recordSize > holder.Length) return result; + var header = 0xAC; + var start = (int)BinaryPrimitives.ReadUInt32LittleEndian(holder.AsSpan(header + 0x00, 4)); + var count = (int)BinaryPrimitives.ReadUInt32LittleEndian(holder.AsSpan(header + 0x14, 4)); + var entries = header + start; + for (var i = 0; i < count && entries + (i + 1) * 24 <= holder.Length; ++i) { + var entry = holder.AsSpan(entries + i * 24, 24); + result.Add(( + BinaryPrimitives.ReadUInt32LittleEndian(entry[0x0C..0x10]), + this.ToPhysical(BinaryPrimitives.ReadUInt64LittleEndian(entry[0x00..0x08])), + BinaryPrimitives.ReadUInt32LittleEndian(entry[0x14..0x18]), + BinaryPrimitives.ReadUInt32LittleEndian(entry[0x08..0x0C]))); + } + return result; + } + + /// Reads a file's bytes by following its decoded extents in the raw image. + public byte[] ReadFileContent(FileView file) { + var result = new byte[file.Size]; + var written = 0; + foreach (var lcn in file.Clusters) { + var take = Math.Min(this.ClusterSize, result.Length - written); + if (take <= 0) break; + this._image.AsSpan((int)((long)lcn * this.ClusterSize), take).CopyTo(result.AsSpan(written)); + written += take; + } + return result; + } + + // ── raw access ───────────────────────────────────────────────────────────── + + public byte[] Cluster(ulong lcn) { + var offset = (long)lcn * this.ClusterSize; + if (offset < 0 || offset + this.ClusterSize > this._image.Length) + throw new InvalidDataException($"Cluster 0x{lcn:X} lies outside the image."); + return this._image.AsSpan((int)offset, this.ClusterSize).ToArray(); + } + + public byte[]? Page(ulong headLcn) { + var offset = (long)headLcn * this.ClusterSize; + if (offset < 0 || offset + this.PageSize > this._image.Length) return null; + return this._image.AsSpan((int)offset, this.PageSize).ToArray(); + } + + private byte[]? PageFromSlots(IReadOnlyList slots) { + var page = new byte[this.PageSize]; + var written = 0; + foreach (var lcn in slots) { + if (written >= page.Length) break; + var offset = (long)lcn * this.ClusterSize; + if (offset < 0 || offset + this.ClusterSize > this._image.Length) return null; + var take = Math.Min(this.ClusterSize, page.Length - written); + this._image.AsSpan((int)offset, take).CopyTo(page.AsSpan(written, take)); + written += take; + } + return written == page.Length ? page : null; + } +} diff --git a/Compression.Tests/Refs/RefsSyntheticVolume.cs b/Compression.Tests/Refs/RefsSyntheticVolume.cs new file mode 100644 index 000000000..c6c628352 --- /dev/null +++ b/Compression.Tests/Refs/RefsSyntheticVolume.cs @@ -0,0 +1,549 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; + +namespace Compression.Tests.Refs; + +/// One file to place into a synthetic ReFS volume. +/// Leaf name inside the root directory. +/// Stream bytes; the allocation is rounded up to a cluster. +/// Number of on-disk extents the stream is split into. +/// Emit a type-0x30/flags-0x01 holder without any decodable +/// extent table, which the namespace reader reports as a resident stream. +internal sealed record RefsSyntheticFile( + string Name, + byte[] Content, + int Fragments = 1, + bool Resident = false); + +/// +/// Builds a byte-exact, structurally valid unmounted ReFS 3.14 image. +/// +/// The package deliberately has no ReFS creator — its writer edits existing +/// volumes only — so the offline block-clone primitive had no image to run +/// against and therefore no executing test. This generator supplies one. It is +/// a test fixture, not a mkfs: it emits the smallest metadata graph that the +/// offline mutation stack needs (VBR, SUPB, two CHKP slots, Container Table, +/// Object Table, Schema Table, Medium Allocator, Block Refcount and a single +/// root-directory object), every tree being one leaf page so no B+ split, no +/// separator-key convention and no Upcase Table are involved. +/// +/// Two fixture choices are worth naming because they differ from a Windows +/// volume: the Container Table publishes one identity container, so VLCN == +/// PLCN throughout; and the root-directory table is stamped with the generic +/// attribute schema 0x120 rather than the filename schema 0x140, so key +/// ordering never needs the 128 KiB Upcase Table. Neither is reachable from +/// the behaviour under test — the cloner never splits a page and never +/// compares filename keys. +/// +internal sealed class RefsSyntheticVolume { + public const int SectorSize = 512; + public const int SectorsPerCluster = 8; + public const int ClusterSize = SectorSize * SectorsPerCluster; + public const int PageSize = 16 * 1024; + public const int ClustersPerPage = PageSize / ClusterSize; + public const int PageReferenceSize = 48; + public const uint ClustersPerContainer = 16384; + public const int TotalClusters = 1024; + public const ulong RootDirectoryOid = 0x600; + public const uint ExtentFlags = 0x180040; + + // Fixed placement. Every metadata page occupies ClustersPerPage clusters. + public const ulong VbrCluster = 0; + public const ulong SuperblockCluster = 0x1E; + public const ulong CheckpointACluster = 36; + public const ulong CheckpointBCluster = 40; + public const ulong ContainerRootCluster = 44; + public const ulong ObjectRootCluster = 48; + public const ulong SchemaRootCluster = 52; + public const ulong MediumAllocatorRootCluster = 56; + public const ulong BlockRefcountRootCluster = 60; + public const ulong DirectoryRootCluster = 64; + public const ulong FirstDataCluster = 68; + + private const int NodeOffset = 0x80; + private const int DataStart = 0xA8; + private const int IndexEnd = PageSize; + private const int SchemaDescriptorOffset = 0x54; + + private readonly List _files = []; + private readonly List _seededRefcountRows = []; + + /// Layout of a built volume: where every file's clusters landed. + internal sealed record Layout( + IReadOnlyDictionary> FileClusters, + IReadOnlyDictionary FileIds, + ulong FirstFreeCluster); + + public Layout? BuiltLayout { get; private set; } + + public RefsSyntheticVolume WithFile(string name, byte[] content, int fragments = 1) { + this._files.Add(new RefsSyntheticFile(name, content, fragments)); + return this; + } + + public RefsSyntheticVolume WithResidentFile(string name, byte[] content) { + this._files.Add(new RefsSyntheticFile(name, content, 1, Resident: true)); + return this; + } + + /// + /// Pre-creates an all-zero Block Refcount row covering . + /// Without it root #6 is empty and the first clone must materialise the row; + /// with it the row exists but the target slot is zero, which is the second + /// first-clone case the codec distinguishes. + /// + public RefsSyntheticVolume WithEmptyRefcountRow(ulong startVirtualLcn = 0) { + this._seededRefcountRows.Add(startVirtualLcn); + return this; + } + + public byte[] Build() { + var image = new byte[(long)TotalClusters * ClusterSize]; + + // ── data placement ─────────────────────────────────────────────────────── + var fileClusters = new Dictionary>(); + var fileExtents = new Dictionary>(); + var fileIds = new Dictionary(); + var cursor = FirstDataCluster; + ulong nextFileId = 0x1000; + + foreach (var file in this._files) { + fileIds[file.Name] = nextFileId++; + if (file.Resident) { + fileClusters[file.Name] = []; + fileExtents[file.Name] = []; + continue; + } + + var clusterCount = (file.Content.Length + ClusterSize - 1) / ClusterSize; + var clusters = new List(clusterCount); + var extents = new List<(ulong Lcn, uint Vcn, uint Run)>(); + if (clusterCount > 0) { + var fragments = Math.Max(1, Math.Min(file.Fragments, clusterCount)); + var perFragment = clusterCount / fragments; + var remainder = clusterCount % fragments; + uint vcn = 0; + for (var f = 0; f < fragments; ++f) { + var run = perFragment + (f < remainder ? 1 : 0); + extents.Add((cursor, vcn, (uint)run)); + for (var i = 0; i < run; ++i) clusters.Add(cursor + (ulong)i); + vcn += (uint)run; + cursor += (ulong)run; + if (f + 1 < fragments) ++cursor; // deliberate hole so the runs stay separate + } + } + fileClusters[file.Name] = clusters; + fileExtents[file.Name] = extents; + + var written = 0; + foreach (var lcn in clusters) { + var take = Math.Min(ClusterSize, file.Content.Length - written); + if (take <= 0) break; + file.Content.AsSpan(written, take).CopyTo(image.AsSpan((int)(lcn * ClusterSize), take)); + written += take; + } + } + + var firstFree = Math.Max(cursor, FirstDataCluster); + this.BuiltLayout = new Layout(fileClusters, fileIds, firstFree); + + // ── bootstrap structures ───────────────────────────────────────────────── + WriteVbr(image); + WriteSuperblock(image); + WriteCheckpoint(image, CheckpointACluster, clock: 2); + WriteCheckpoint(image, CheckpointBCluster, clock: 1); + + // ── metadata trees ─────────────────────────────────────────────────────── + WritePage(image, ContainerRootCluster, tableId: 0x0B, schemaId: 0xE0C0, physical: true, + rows: [new Row(ContainerKey(), ContainerValue())]); + + WritePage(image, ObjectRootCluster, tableId: 0x02, schemaId: 0xE030, physical: false, + rows: [new Row(ObjectKey(RootDirectoryOid), ObjectValue(DirectoryRootCluster))]); + + WritePage(image, SchemaRootCluster, tableId: 0x03, schemaId: 0xE060, physical: false, + rows: SchemaRows()); + + WritePage(image, MediumAllocatorRootCluster, tableId: 0x21, schemaId: 0xE010, physical: false, + rows: [new Row(AllocatorKey(), AllocatorValue(firstFree))]); + + WritePage(image, BlockRefcountRootCluster, tableId: 0x0F, schemaId: 0xE0B0, physical: false, + rows: this._seededRefcountRows + .Distinct() + .OrderBy(start => start) + .Select(start => new Row(RefcountKey(start), RefcountValue(start))) + .ToList()); + + WritePage(image, DirectoryRootCluster, tableId: 0x300, schemaId: 0x120, physical: false, + rows: this.DirectoryRows(fileIds, fileExtents)); + + return image; + } + + // ── bootstrap ────────────────────────────────────────────────────────────── + + private static void WriteVbr(byte[] image) { + var vbr = image.AsSpan(0, 512); + Encoding.ASCII.GetBytes("ReFS").CopyTo(vbr[3..]); + Encoding.ASCII.GetBytes("FSRS").CopyTo(vbr[0x10..]); + BinaryPrimitives.WriteUInt16LittleEndian(vbr.Slice(0x14, 2), 0x200); + BinaryPrimitives.WriteUInt64LittleEndian(vbr.Slice(0x18, 8), (ulong)TotalClusters * SectorsPerCluster); + BinaryPrimitives.WriteUInt32LittleEndian(vbr.Slice(0x20, 4), SectorSize); + BinaryPrimitives.WriteUInt32LittleEndian(vbr.Slice(0x24, 4), SectorsPerCluster); + vbr[0x28] = 3; + vbr[0x29] = 14; + BinaryPrimitives.WriteUInt16LittleEndian(vbr.Slice(0x2A, 2), 2); + BinaryPrimitives.WriteUInt32LittleEndian(vbr.Slice(0x2C, 4), 0x66); + BinaryPrimitives.WriteUInt64LittleEndian(vbr.Slice(0x38, 8), 0x0123456789ABCDEFUL); + BinaryPrimitives.WriteUInt64LittleEndian(vbr.Slice(0x40, 8), (ulong)ClustersPerContainer * ClusterSize); + BinaryPrimitives.WriteUInt16LittleEndian(vbr.Slice(0x16, 2), VbrChecksum(vbr)); + } + + internal static ushort VbrChecksum(ReadOnlySpan vbr) { + ushort sum = 0; + for (var i = 3; i < 512; ++i) { + if (i is 0x16 or 0x17) continue; + sum = (ushort)((sum >> 1) | (sum << 15)); + sum = unchecked((ushort)(sum + vbr[i])); + } + return sum; + } + + private static void WriteSuperblock(byte[] image) { + var supb = image.AsSpan((int)(SuperblockCluster * ClusterSize), ClusterSize); + Encoding.ASCII.GetBytes("SUPB").CopyTo(supb); + BinaryPrimitives.WriteUInt64LittleEndian(supb.Slice(0x68, 8), 1); + BinaryPrimitives.WriteUInt32LittleEndian(supb.Slice(0x70, 4), 0x80); + BinaryPrimitives.WriteUInt32LittleEndian(supb.Slice(0x74, 4), 2); + BinaryPrimitives.WriteUInt32LittleEndian(supb.Slice(0x78, 4), 0x100); + BinaryPrimitives.WriteUInt32LittleEndian(supb.Slice(0x7C, 4), PageReferenceSize); + BinaryPrimitives.WriteUInt64LittleEndian(supb.Slice(0x80, 8), CheckpointACluster); + BinaryPrimitives.WriteUInt64LittleEndian(supb.Slice(0x88, 8), CheckpointBCluster); + // Self-checksum descriptor: type 0 / length 0, so every Refresh* call is a + // no-op. Nothing on the ReFS read path verifies a digest. + } + + private static void WriteCheckpoint(byte[] image, ulong headCluster, ulong clock) { + var page = new byte[PageSize]; + Encoding.ASCII.GetBytes("CHKP").CopyTo(page.AsSpan()); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x10, 8), clock); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x20, 8), headCluster); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x58, 4), 0x380); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x5C, 4), PageReferenceSize); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x60, 8), clock); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x68, 8), clock); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x78, 4), 0x0002); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x90, 4), 13); + for (var i = 0; i < 13; ++i) + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x94 + i * 4, 4), (uint)(0x100 + i * PageReferenceSize)); + + StampRoot(page, 0, ObjectRootCluster); + StampRoot(page, 1, MediumAllocatorRootCluster); + StampRoot(page, 3, SchemaRootCluster); + StampRoot(page, 6, BlockRefcountRootCluster); + StampRoot(page, 7, ContainerRootCluster); + + WriteClusters(image, headCluster, page); + } + + private static void StampRoot(byte[] checkpoint, int rootIndex, ulong headCluster) { + var descriptor = 0x100 + rootIndex * PageReferenceSize; + for (var i = 0; i < ClustersPerPage; ++i) + BinaryPrimitives.WriteUInt64LittleEndian( + checkpoint.AsSpan(descriptor + i * 8, 8), + headCluster + (ulong)i); + // reference[0x22] = checksum type 0, reference[0x24] = digest length 0. + } + + private static void WriteClusters(byte[] image, ulong headCluster, ReadOnlySpan page) { + for (var i = 0; i < page.Length; i += ClusterSize) + page.Slice(i, Math.Min(ClusterSize, page.Length - i)) + .CopyTo(image.AsSpan((int)(headCluster * ClusterSize) + i)); + } + + // ── B+ leaf pages ────────────────────────────────────────────────────────── + + internal sealed record Row(byte[] Key, byte[] Value); + + private static void WritePage( + byte[] image, + ulong headCluster, + ulong tableId, + ushort schemaId, + bool physical, + IReadOnlyList rows) { + var page = new byte[PageSize]; + Encoding.ASCII.GetBytes("MSB+").CopyTo(page.AsSpan()); + for (var i = 0; i < ClustersPerPage; ++i) + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x20 + i * 8, 8), headCluster + (ulong)i); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x48, 8), tableId); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x50, 4), NodeOffset - 0x50); + + // IndexRoot descriptor: RefsCowBTree.FindRootSchema scans [0x50, nodeOffset) + // for a 0x28-byte descriptor and reads the schema id from +0x0C. + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(SchemaDescriptorOffset, 4), 0x28); + BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(SchemaDescriptorOffset + 0x0C, 2), schemaId); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(SchemaDescriptorOffset + 0x20, 8), (ulong)rows.Count); + + var serialized = rows.Select(SerializeRow).ToArray(); + var rowBytes = serialized.Sum(r => r.Length); + var indexStart = IndexEnd - rows.Count * 4; + if (DataStart + rowBytes > indexStart) + throw new InvalidOperationException("Synthetic ReFS page overflows; the fixture needs a smaller tree."); + + var offset = DataStart; + for (var i = 0; i < serialized.Length; ++i) { + serialized[i].CopyTo(page, offset); + BinaryPrimitives.WriteUInt32LittleEndian( + page.AsSpan(indexStart + i * 4, 4), + 0xFFFF0000U | (uint)(offset - NodeOffset)); + offset += serialized[i].Length; + } + + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x00, 4), DataStart - NodeOffset); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x04, 4), (uint)(offset - NodeOffset)); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x08, 4), (uint)(indexStart - offset)); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x0C, 4), 0); // leaf + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x10, 4), (uint)(indexStart - NodeOffset)); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x14, 4), (uint)rows.Count); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x20, 4), IndexEnd - NodeOffset); + + WriteClusters(image, headCluster, page); + } + + private static byte[] SerializeRow(Row row) { + const int keyOffset = 16; + var valueOffset = Align8(keyOffset + row.Key.Length); + var rowSize = Align8(valueOffset + row.Value.Length); + var result = new byte[rowSize]; + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(0, 4), (uint)rowSize); + BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(4, 2), keyOffset); + BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(6, 2), (ushort)row.Key.Length); + BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(10, 2), (ushort)valueOffset); + BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(12, 2), (ushort)row.Value.Length); + row.Key.CopyTo(result, keyOffset); + row.Value.CopyTo(result, valueOffset); + return result; + } + + private static int Align8(int value) => (value + 7) & ~7; + + // ── row builders ─────────────────────────────────────────────────────────── + + private static byte[] ContainerKey() => new byte[16]; + + private static byte[] ContainerValue() { + var value = new byte[0x98]; + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x18, 4), ClustersPerContainer); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(value.Length - 16, 8), 0UL); + return value; + } + + private static byte[] ObjectKey(ulong oid) { + var key = new byte[16]; + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), oid); + return key; + } + + private static byte[] ObjectValue(ulong rootHeadCluster) { + var value = new byte[0x20 + PageReferenceSize]; + for (var i = 0; i < ClustersPerPage; ++i) + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x20 + i * 8, 8), rootHeadCluster + (ulong)i); + return value; + } + + private static readonly uint[] SchemaIds = [ + 0x110, 0x120, 0x130, 0x140, + 0xE010, 0xE030, 0xE040, 0xE060, 0xE080, 0xE090, 0xE0B0, 0xE0C0, + ]; + + private static List SchemaRows() { + var rows = new List(SchemaIds.Length); + foreach (var schemaId in SchemaIds.OrderBy(id => id)) { + var key = new byte[8]; + BinaryPrimitives.WriteUInt32LittleEndian(key.AsSpan(0, 4), schemaId); + var value = new byte[0x50]; + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x00, 4), 0x50); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x04, 4), 0x18); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x18, 4), 0x38); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x1C, 4), 1); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x24, 4), schemaId); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x48, 4), 1); + rows.Add(new Row(key, value)); + } + return rows; + } + + private static byte[] AllocatorKey() { + var key = new byte[16]; + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(0, 8), 0UL); + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), TotalClusters); + return key; + } + + /// + /// One bitmap Medium Allocator row covering the whole volume. Everything below + /// is metadata or file data; the rest is the pool + /// the CoW page store reserves its replacement pages from. + /// + private static byte[] AllocatorValue(ulong firstFree) { + var value = new byte[0x18 + 2048]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x00, 8), 0UL); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x08, 8), TotalClusters); + + var used = 0; + for (ulong lcn = 0; lcn < firstFree; ++lcn) { + value[0x18 + (int)(lcn >> 3)] |= (byte)(1 << (int)(lcn & 7)); + ++used; + } + + BinaryPrimitives.WriteUInt16LittleEndian(value.AsSpan(0x10, 2), (ushort)(TotalClusters - used)); + BinaryPrimitives.WriteUInt16LittleEndian(value.AsSpan(0x12, 2), 0x01); + BinaryPrimitives.WriteUInt16LittleEndian(value.AsSpan(0x14, 2), 0x0218); + BinaryPrimitives.WriteUInt16LittleEndian(value.AsSpan(0x16, 2), (ushort)used); + return value; + } + + private static byte[] RefcountKey(ulong startVirtualLcn) { + var key = new byte[16]; + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(0, 8), startVirtualLcn); + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), 0x400); + return key; + } + + private static byte[] RefcountValue(ulong startVirtualLcn) { + var value = new byte[0x820]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x00, 8), startVirtualLcn); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x08, 8), 0x400); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x10, 8), 1UL); + return value; + } + + private List DirectoryRows( + IReadOnlyDictionary fileIds, + IReadOnlyDictionary> fileExtents) { + var rows = new List(); + foreach (var file in this._files) { + var fileId = fileIds[file.Name]; + var size = (ulong)file.Content.Length; + var allocated = (ulong)(((file.Content.Length + ClusterSize - 1) / ClusterSize) * (long)ClusterSize); + + if (file.Resident) { + rows.Add(new Row(NameKey(file.Name, 0x01), ResidentHolderValue(size))); + continue; + } + + rows.Add(new Row(NameKey(file.Name, 0x02), FileRecordValue(fileId, size, allocated))); + rows.Add(new Row(BackingKey(fileId), BackingValue(size, allocated, fileExtents[file.Name]))); + } + rows.Sort((a, b) => CompareAttributeKey(a.Key, b.Key)); + return rows; + } + + private static byte[] NameKey(string name, ushort keyFlags) { + var encoded = Encoding.Unicode.GetBytes(name); + var key = new byte[4 + encoded.Length]; + BinaryPrimitives.WriteUInt16LittleEndian(key.AsSpan(0, 2), 0x30); + BinaryPrimitives.WriteUInt16LittleEndian(key.AsSpan(2, 2), keyFlags); + encoded.CopyTo(key, 4); + return key; + } + + private static byte[] FileRecordValue(ulong fileId, ulong size, ulong allocated) { + var value = new byte[0x48]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x00, 8), fileId); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x08, 8), RootDirectoryOid); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x18, 8), 0x01D0_0000_0000_0000UL); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x30, 8), allocated); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x38, 8), size); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x40, 4), 0x20); + return value; + } + + private static byte[] ResidentHolderValue(ulong size) { + var value = new byte[0x80]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x30, 8), 0x01D0_0000_0000_0000UL); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x48, 4), 0x20); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x58, 8), size); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x60, 8), 0UL); + return value; + } + + private static byte[] BackingKey(ulong fileId) { + var key = new byte[16]; + BinaryPrimitives.WriteUInt16LittleEndian(key.AsSpan(0, 2), 0x40); + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), fileId); + return key; + } + + // Native 3.10+ extent-holder value. The sub-record starts at +0xA8, its extent + // table header at +0xAC and the 24-byte entries at +0xD0 — the layout the + // offline writer's TryRewriteNativeHolder recognises and rewrites in place. + public const int BackingSubRecordOffset = 0xA8; + public const int BackingTableHeaderOffset = 0xAC; + public const int BackingEntriesOffset = 0xD0; + + internal static byte[] BackingValue( + ulong size, + ulong allocated, + IReadOnlyList<(ulong Lcn, uint Vcn, uint Run)> extents) { + var length = BackingEntriesOffset + extents.Count * 24; + var value = new byte[Math.Max(length, 0x68)]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x58, 8), size); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x60, 8), allocated); + if (extents.Count == 0) return value; + + const int start = BackingEntriesOffset - BackingTableHeaderOffset; + BinaryPrimitives.WriteUInt32LittleEndian( + value.AsSpan(BackingSubRecordOffset, 4), (uint)(length - BackingSubRecordOffset)); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(BackingTableHeaderOffset + 0x00, 4), start); + BinaryPrimitives.WriteUInt32LittleEndian( + value.AsSpan(BackingTableHeaderOffset + 0x04, 4), (uint)(start + extents.Count * 24)); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(BackingTableHeaderOffset + 0x0C, 4), 0xE00); + BinaryPrimitives.WriteUInt32LittleEndian( + value.AsSpan(BackingTableHeaderOffset + 0x14, 4), (uint)extents.Count); + + for (var i = 0; i < extents.Count; ++i) { + var entry = value.AsSpan(BackingEntriesOffset + i * 24, 24); + BinaryPrimitives.WriteUInt64LittleEndian(entry[0x00..0x08], extents[i].Lcn); + BinaryPrimitives.WriteUInt32LittleEndian(entry[0x08..0x0C], ExtentFlags); + BinaryPrimitives.WriteUInt32LittleEndian(entry[0x0C..0x10], extents[i].Vcn); + BinaryPrimitives.WriteUInt32LittleEndian(entry[0x14..0x18], extents[i].Run); + } + return value; + } + + /// + /// Replica of the key ordering ReFS uses for the generic attribute schemas: + /// u16 type, u16 flags, then little-endian u64 chunks, then a byte tail. The + /// fixture must emit rows in exactly this order or the CoW engine rejects the + /// tree before it mutates anything. + /// + internal static int CompareAttributeKey(ReadOnlySpan left, ReadOnlySpan right) { + if (left.Length >= 4 && right.Length >= 4) { + var type = BinaryPrimitives.ReadUInt16LittleEndian(left[..2]) + .CompareTo(BinaryPrimitives.ReadUInt16LittleEndian(right[..2])); + if (type != 0) return type; + var flags = BinaryPrimitives.ReadUInt16LittleEndian(left.Slice(2, 2)) + .CompareTo(BinaryPrimitives.ReadUInt16LittleEndian(right.Slice(2, 2))); + if (flags != 0) return flags; + } + + var cursor = 4; + while (cursor + 8 <= left.Length && cursor + 8 <= right.Length) { + var cmp = BinaryPrimitives.ReadUInt64LittleEndian(left.Slice(cursor, 8)) + .CompareTo(BinaryPrimitives.ReadUInt64LittleEndian(right.Slice(cursor, 8))); + if (cmp != 0) return cmp; + cursor += 8; + } + + var tailLeft = left[cursor..]; + var tailRight = right[cursor..]; + var count = Math.Min(tailLeft.Length, tailRight.Length); + for (var i = 0; i < count; ++i) { + var cmp = tailLeft[i].CompareTo(tailRight[i]); + if (cmp != 0) return cmp; + } + return tailLeft.Length.CompareTo(tailRight.Length); + } +} diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index 387b5e579..b160fab64 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -330,7 +330,7 @@ cwb recover raw.img --mode files --format Jpeg,Png - This is an image-manipulation toolkit, not a kernel filesystem driver. R/W does not imply concurrent mount semantics, crash-consistent journaling under arbitrary interruption, or a drop-in replacement for the OS driver. - Where **Notes** says a format's edits *rebuild* the volume, an add or remove costs the whole image rather than the bytes that changed, and the block placement of untouched files may move. - **bcachefs** mutation is profile-gated: the in-place writer owns single-device regular pointer extents as emitted by this package. Bucket generations are carried through a mutation: emptying a bucket advances its generation, and the alloc key, the `bucket_gens` slot, the extent pointer and its backpointer are written from one source and checked against each other before the commit is published. Foreign volumes with extra live b-trees, non-canonical `bucket_gens` or `alloc_v4` keys, inline / reflink / compressed extent keys, or unsupported inode and dirent types are refused for mutation rather than rewritten speculatively; read support is broader. How the accounting, backpointer and LRU keys were established is written up in [BCACHEFS-ACCOUNTING.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/BCACHEFS-ACCOUNTING.md). -- **ReFS** edits are offline-quiescent: the image must not be mounted, and `RefsMutationMode.NativeCow` stays fail-closed. The reader walks ReFS 3.x containers, MSB+ trees, allocators, Block Refcount rows and MLog framing; the offline editor relocates data and metadata pages, replaces and removes regular files and empty directories through CoW B+ replacement and alternate-checkpoint publication. Still open before a mounted driver could use the same core: redo-payload codecs and replay for every opcode, every Schema Table key rule, the allocation-zone policies of the three allocators, container create / delete / move, Block Refcount row creation and clone semantics, hard links, sparse and integrity-stream mutation beyond 4 KiB clusters, snapshots, ADS, security descriptors, reparse points, USN, rename, truncate, a format path, locking and a Windows `chkdsk` corpus. +- **ReFS** edits are offline-quiescent: the image must not be mounted, and `RefsMutationMode.NativeCow` stays fail-closed. The reader walks ReFS 3.x containers, MSB+ trees, allocators, Block Refcount rows and MLog framing; the offline editor relocates data and metadata pages, replaces and removes regular files and empty directories through CoW B+ replacement and alternate-checkpoint publication. Block Refcount row creation and whole-file clone semantics are implemented and tested: `RefsOfflineBlockCloner.CloneWholeFile` repoints an existing destination stream at an existing, equal-sized, cluster-aligned source stream, materialises the shared-cluster reference counts — including the first clone of a slot the sparse table does not yet track — and publishes Object Table root #0 and Block Refcount root #6 through one alternate checkpoint, reclaiming the destination's former allocation only once that checkpoint is durable. Writing to a clone afterwards reallocates rather than overwriting the shared clusters. **No public verb reaches this primitive**: it is a library entry point, not something `cwb` can be asked to do, and the profile is deliberately narrower than `FSCTL_DUPLICATE_EXTENTS_TO_FILE` — resident, sparse, integrity, zero-length, already-shared and partial-range clones are all refused. Still open before a mounted driver could use the same core: redo-payload codecs and replay for every opcode, every Schema Table key rule, the allocation-zone policies of the three allocators, container create / delete / move, partial-range and EOF-extending clones, hard links, sparse and integrity-stream mutation beyond 4 KiB clusters, snapshots, ADS, security descriptors, reparse points, USN, rename, truncate, a format path, locking and a Windows `chkdsk` corpus. - **Reiser4** remains WORM: the current writer captures the empty native tree but stores added files in a payload area the reiser4 driver does not see. It must not be promoted until the native tree writer owns those files. - **GFS2** R/W is the standalone `lock_nolock` profile with regular files in the root directory; small files are stuffed and larger files use the indirect tree. Existing-image edits rebuild the volume while preserving its size floor and lock-table value. ExHash/nested-directory writing, journal replay, cluster-lock-manager state and extended attributes remain out of scope. - **NSS** native read support is intentionally narrow and quiescent: the reader reconstructs validated DirH/LEAF namespace relationships and ordinary contiguous file extents using the OES-derived clean-room profile in [NSS-ON-DISK.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/NSS-ON-DISK.md), while retaining anchor diagnostics for unsupported images. Native allocation trees, journals, trustee metadata, ZLBH history, compressed/sparse/fragmented data profiles and writable NSS pools remain out of scope; the existing writer still emits a private Workbench container rather than a pool NetWare/OES would mount. From 307d979566cb3e5e5e6310b4bf3522e25618dac1 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Mon, 21 Sep 2026 11:17:24 +0200 Subject: [PATCH 2/2] - the ReFS diagnostic workflow pinned to a deleted branch .github/workflows/refs-diagnostic.yml triggered only on pushes to the branch refs-block-refcount-clone, which no longer exists: its content was absorbed into #325 and the remote branch has since been deleted, so the workflow can never fire again. It was never a gate in the first place. The job cloned the PNGCrushCS sibling, restored and built the solution with continue-on-error, uploaded the build log and re-raised the build result - a debugging aid for one red branch. That branch was red three times on error CS0104: 'Category' is an ambiguous reference between System.ComponentModel.CategoryAttribute and NUnit.Framework.CategoryAttribute which #325 fixed with a using alias, leaving the workflow with nothing to diagnose. No other workflow or script references the dead branch. --- .github/workflows/refs-diagnostic.yml | 57 --------------------------- 1 file changed, 57 deletions(-) delete mode 100644 .github/workflows/refs-diagnostic.yml diff --git a/.github/workflows/refs-diagnostic.yml b/.github/workflows/refs-diagnostic.yml deleted file mode 100644 index 3470908a9..000000000 --- a/.github/workflows/refs-diagnostic.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: ReFS diagnostic - -on: - push: - branches: [refs-block-refcount-clone] - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Clone PNGCrushCS sibling - run: git clone --depth 1 https://github.com/Hawkynt/PNGCrushCS.git "$GITHUB_WORKSPACE/../PNGCrushCS" - - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '10.0.x' - dotnet-quality: preview - - name: Pre-build sibling - shell: bash - run: | - set -euo pipefail - cd "$GITHUB_WORKSPACE/../PNGCrushCS" - REQUIRED="Compression.Core FileFormat.Core FileFormat.Bmp FileFormat.Apng FileFormat.Tiff FileFormat.BigTiff FileFormat.Mng FileFormat.Fli FileFormat.Dcx FileFormat.Icns FileFormat.Mpo FileFormat.Jpeg FileFormat.Png" - locate() { - local proj="$1" - if [ -f "$proj/$proj.csproj" ]; then echo "$proj"; return 0; fi - if [ -f "FileFormats/$proj/$proj.csproj" ]; then echo "FileFormats/$proj"; return 0; fi - return 1 - } - for proj in $REQUIRED; do - if ! locate "$proj" >/dev/null; then exit 0; fi - done - for proj in $REQUIRED; do - dir="$(locate "$proj")" - dotnet build "$dir/$proj.csproj" --configuration Release --nologo - done - - name: Restore - run: dotnet restore CompressionWorkbench.slnx - - name: Build and capture diagnostics - id: build - shell: bash - continue-on-error: true - run: | - set -o pipefail - dotnet build CompressionWorkbench.slnx --configuration Release --no-restore 2>&1 | tee refs-build.log - - uses: actions/upload-artifact@v4 - with: - name: refs-build-log - path: refs-build.log - - name: Propagate build result - if: steps.build.outcome != 'success' - run: exit 1