diff --git a/.github/workflows/refs-diagnostic.yml b/.github/workflows/refs-diagnostic.yml deleted file mode 100644 index 3470908a9..000000000 --- a/.github/workflows/refs-diagnostic.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: ReFS diagnostic - -on: - push: - branches: [refs-block-refcount-clone] - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Clone PNGCrushCS sibling - run: git clone --depth 1 https://github.com/Hawkynt/PNGCrushCS.git "$GITHUB_WORKSPACE/../PNGCrushCS" - - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '10.0.x' - dotnet-quality: preview - - name: Pre-build sibling - shell: bash - run: | - set -euo pipefail - cd "$GITHUB_WORKSPACE/../PNGCrushCS" - REQUIRED="Compression.Core FileFormat.Core FileFormat.Bmp FileFormat.Apng FileFormat.Tiff FileFormat.BigTiff FileFormat.Mng FileFormat.Fli FileFormat.Dcx FileFormat.Icns FileFormat.Mpo FileFormat.Jpeg FileFormat.Png" - locate() { - local proj="$1" - if [ -f "$proj/$proj.csproj" ]; then echo "$proj"; return 0; fi - if [ -f "FileFormats/$proj/$proj.csproj" ]; then echo "FileFormats/$proj"; return 0; fi - return 1 - } - for proj in $REQUIRED; do - if ! locate "$proj" >/dev/null; then exit 0; fi - done - for proj in $REQUIRED; do - dir="$(locate "$proj")" - dotnet build "$dir/$proj.csproj" --configuration Release --nologo - done - - name: Restore - run: dotnet restore CompressionWorkbench.slnx - - name: Build and capture diagnostics - id: build - shell: bash - continue-on-error: true - run: | - set -o pipefail - dotnet build CompressionWorkbench.slnx --configuration Release --no-restore 2>&1 | tee refs-build.log - - uses: actions/upload-artifact@v4 - with: - name: refs-build-log - path: refs-build.log - - name: Propagate build result - if: steps.build.outcome != 'success' - run: exit 1 diff --git a/Compression.Tests/Refs/RefsBlockCloneTests.cs b/Compression.Tests/Refs/RefsBlockCloneTests.cs new file mode 100644 index 000000000..522986a3a --- /dev/null +++ b/Compression.Tests/Refs/RefsBlockCloneTests.cs @@ -0,0 +1,471 @@ +#pragma warning disable CS1591 +using Compression.Registry; +using FileSystem.Refs; + +namespace Compression.Tests.Refs; + +/// +/// Executing coverage for the offline ReFS whole-file block clone. +/// +/// and +/// previously had no test that ran: their only caller was an external-oracle +/// fixture that skips unless two environment variables point at a real ReFS +/// image, which CI never sets. A wrong block clone corrupts across files +/// silently — two streams sharing clusters they should not, or reference counts +/// that disagree with reality — so the primitive is driven here against images +/// built by and checked with +/// , which decodes the resulting bytes without going +/// back through the code that wrote them. +/// +[TestFixture] +public sealed class RefsBlockCloneTests { + private const int ClusterSize = RefsSyntheticVolume.ClusterSize; + + private static byte[] Pattern(byte seed, int length) { + var result = new byte[length]; + for (var i = 0; i < length; ++i) result[i] = (byte)(seed + i * 31); + return result; + } + + [Test, Category("HappyPath")] + public void SyntheticVolume_IsReadableAsARefsNamespace() { + var source = Pattern(0x11, 2 * ClusterSize); + var destination = Pattern(0x77, 2 * ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", source) + .WithFile("bravo.bin", destination) + .Build(); + + using var stream = new MemoryStream(image, writable: true); + var metadata = RefsMetadataReader.Open(stream); + var files = new RefsNamespaceReader(metadata).ReadAll(); + + Assert.Multiple(() => { + Assert.That(files.Select(f => f.Path), Is.EquivalentTo(new[] { "alpha.bin", "bravo.bin" })); + var alpha = files.Single(f => f.Path == "alpha.bin"); + Assert.That(alpha.IsResident, Is.False); + Assert.That(alpha.Size, Is.EqualTo(source.Length)); + Assert.That(alpha.AllocatedSize, Is.EqualTo(source.Length)); + Assert.That(alpha.Extents, Has.Count.EqualTo(1)); + Assert.That(alpha.Extents[0].ClusterCount, Is.EqualTo(2)); + }); + + var probe = new RefsImageProbe(image); + var probed = probe.ReadFiles(probe.ActiveCheckpoint()); + Assert.Multiple(() => { + Assert.That(probe.ReadFileContent(probed["alpha.bin"]), Is.EqualTo(source)); + Assert.That(probe.ReadFileContent(probed["bravo.bin"]), Is.EqualTo(destination)); + }); + } + + [Test, Category("HappyPath")] + public void WholeFileClone_SharesTheSourceClustersAndMaterialisesCountTwo() { + var sourceBytes = Pattern(0x11, 2 * ClusterSize); + var destinationBytes = Pattern(0x77, 2 * ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", sourceBytes) + .WithFile("bravo.bin", destinationBytes) + .Build(); + + var before = new RefsImageProbe(image); + var beforeCheckpoint = before.ActiveCheckpoint(); + var beforeFiles = before.ReadFiles(beforeCheckpoint); + var sourceClusters = beforeFiles["alpha.bin"].Clusters; + var formerDestinationClusters = beforeFiles["bravo.bin"].Clusters; + Assume.That(before.CountRefcountRows(beforeCheckpoint), Is.Zero, + "the fixture must start with a sparse Block Refcount tree"); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var afterFiles = after.ReadFiles(afterCheckpoint); + var refcounts = after.ReadRefcounts(afterCheckpoint); + var allocated = after.ReadAllocatedClusters(afterCheckpoint); + + Assert.Multiple(() => { + // Structural: both stream layouts now name the same physical clusters. + Assert.That(afterFiles["alpha.bin"].Clusters, Is.EqualTo(sourceClusters)); + Assert.That(afterFiles["bravo.bin"].Clusters, Is.EqualTo(sourceClusters)); + Assert.That(afterFiles["bravo.bin"].Size, Is.EqualTo((ulong)sourceBytes.Length)); + Assert.That(afterFiles["bravo.bin"].AllocatedSize, Is.EqualTo((ulong)sourceBytes.Length)); + + // Structural: every shared cluster carries reference count 2. + foreach (var cluster in sourceClusters) { + Assert.That(refcounts.ContainsKey(cluster), Is.True, + $"PLCN 0x{cluster:X} has no Block Refcount row after the clone."); + Assert.That(refcounts[cluster].Count, Is.EqualTo(2), + $"PLCN 0x{cluster:X} is not shared by exactly two streams."); + } + + // Structural: the row's own total field agrees with its words. + foreach (var (start, totals) in after.ReadRefcountTotals(afterCheckpoint)) + Assert.That(totals.Stated, Is.EqualTo(totals.Actual), + $"Block Refcount row 0x{start:X} states a total its words do not add up to."); + + // Round-trip: both names read back as the source content. + Assert.That(after.ReadFileContent(afterFiles["alpha.bin"]), Is.EqualTo(sourceBytes)); + Assert.That(after.ReadFileContent(afterFiles["bravo.bin"]), Is.EqualTo(sourceBytes)); + + // Structural: the destination's former allocation went back to the pool. + foreach (var cluster in formerDestinationClusters) + Assert.That(allocated, Does.Not.Contain(cluster), + $"The destination's former PLCN 0x{cluster:X} is still marked allocated."); + foreach (var cluster in sourceClusters) + Assert.That(allocated, Does.Contain(cluster), + $"Shared PLCN 0x{cluster:X} must stay allocated."); + }); + } + + [Test, Category("HappyPath")] + public void FirstClone_OnAZeroSlotOfAnExistingRow_BecomesTwoWithoutAddingARow() { + var sourceBytes = Pattern(0x21, ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", sourceBytes) + .WithFile("bravo.bin", Pattern(0x5A, ClusterSize)) + .WithEmptyRefcountRow() + .Build(); + + var before = new RefsImageProbe(image); + var beforeCheckpoint = before.ActiveCheckpoint(); + var sourceClusters = before.ReadFiles(beforeCheckpoint)["alpha.bin"].Clusters; + Assert.Multiple(() => { + Assert.That(before.CountRefcountRows(beforeCheckpoint), Is.EqualTo(1), + "the fixture must start with exactly one all-zero Block Refcount row"); + Assert.That(before.ReadRefcounts(beforeCheckpoint), Is.Empty, + "no slot of the seeded row may carry a non-zero word yet"); + }); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var refcounts = after.ReadRefcounts(afterCheckpoint); + + Assert.Multiple(() => { + Assert.That(after.CountRefcountRows(afterCheckpoint), Is.EqualTo(1), + "an existing row must be updated, not duplicated"); + foreach (var cluster in sourceClusters) + Assert.That(refcounts[cluster].Count, Is.EqualTo(2), + $"A zero slot must become 2 on first sharing, not 1 — PLCN 0x{cluster:X}."); + Assert.That(refcounts, Has.Count.EqualTo(sourceClusters.Count), + "only the shared clusters may gain a non-zero reference word"); + foreach (var (start, totals) in after.ReadRefcountTotals(afterCheckpoint)) + Assert.That(totals.Stated, Is.EqualTo(totals.Actual), + $"Block Refcount row 0x{start:X} states a stale total."); + }); + } + + [Test, Category("HappyPath")] + public void Clone_PublishesObjectTableAndRefcountRootsInOneAlternateCheckpoint() { + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", Pattern(0x31, 2 * ClusterSize)) + .WithFile("bravo.bin", Pattern(0x8F, 2 * ClusterSize)) + .Build(); + + var before = new RefsImageProbe(image); + var beforeCheckpoint = before.ActiveCheckpoint(); + var beforeFiles = before.ReadFiles(beforeCheckpoint); + var formerDestinationClusters = beforeFiles["bravo.bin"].Clusters; + var beforeAllocated = before.ReadAllocatedClusters(beforeCheckpoint); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var stale = after.ReadCheckpoint(beforeCheckpoint.Lcn); + var afterAllocated = after.ReadAllocatedClusters(afterCheckpoint); + + Assert.Multiple(() => { + // The publication is an alternate-slot swap, not an in-place edit. + Assert.That(afterCheckpoint.Lcn, Is.Not.EqualTo(beforeCheckpoint.Lcn)); + Assert.That(afterCheckpoint.Clock, Is.GreaterThan(beforeCheckpoint.Clock)); + + // Object Table #0 and Block Refcount #6 moved in the same publication; + // a checkpoint carrying one without the other is the silent-corruption + // case this primitive exists to avoid. + Assert.That(afterCheckpoint.Roots[0], Is.Not.EqualTo(beforeCheckpoint.Roots[0]), + "Object Table root #0 did not move."); + Assert.That(afterCheckpoint.Roots[6], Is.Not.EqualTo(beforeCheckpoint.Roots[6]), + "Block Refcount root #6 did not move."); + Assert.That(afterCheckpoint.Roots[1], Is.Not.EqualTo(beforeCheckpoint.Roots[1]), + "Medium Allocator root #1 must account for the newly reserved CoW pages."); + Assert.That(afterCheckpoint.Roots[7], Is.EqualTo(beforeCheckpoint.Roots[7]), + "The Container Table has no business changing during a block clone."); + + // The superseded slot is untouched and still describes the pre-clone volume. + Assert.That(stale.Clock, Is.EqualTo(beforeCheckpoint.Clock)); + Assert.That(stale.Roots[0], Is.EqualTo(beforeCheckpoint.Roots[0])); + Assert.That(stale.Roots[6], Is.EqualTo(beforeCheckpoint.Roots[6])); + Assert.That(after.ReadFiles(stale)["bravo.bin"].Clusters, Is.EqualTo(formerDestinationClusters)); + Assert.That(after.CountRefcountRows(stale), Is.Zero); + + // Every page the replacement roots point at is newly reserved storage that + // the published allocator now owns. + foreach (var rootIndex in new[] { 0, 1, 6 }) + foreach (var lcn in afterCheckpoint.Roots[rootIndex]) { + var physical = after.ToPhysical(lcn); + Assert.That(beforeAllocated, Does.Not.Contain(physical), + $"Root #{rootIndex} was rebuilt over live PLCN 0x{physical:X}."); + Assert.That(afterAllocated, Does.Contain(physical), + $"Root #{rootIndex} page PLCN 0x{physical:X} is not marked allocated by the published allocator."); + } + + // Reclamation is visible only through the new checkpoint. + foreach (var cluster in formerDestinationClusters) { + Assert.That(beforeAllocated, Does.Contain(cluster)); + Assert.That(afterAllocated, Does.Not.Contain(cluster)); + } + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_AbortedWhilePublishing_LeavesTheVolumeExactlyAsItWas() { + var sourceBytes = Pattern(0x41, 2 * ClusterSize); + var destinationBytes = Pattern(0x9D, 2 * ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", sourceBytes) + .WithFile("bravo.bin", destinationBytes) + .Build(); + + var before = new RefsImageProbe(image); + var beforeCheckpoint = before.ActiveCheckpoint(); + var beforeFiles = before.ReadFiles(beforeCheckpoint); + var beforeAllocated = before.ReadAllocatedClusters(beforeCheckpoint); + var alternate = before.CheckpointLcns.Single(lcn => lcn != beforeCheckpoint.Lcn); + + using (var stream = new WriteBarrierStream( + image, + (long)alternate * ClusterSize, + (long)(alternate + (ulong)before.ClustersPerPage) * ClusterSize)) { + Assert.Throws(() => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin")); + Assert.That(stream.Tripped, Is.True, "the abort must happen while the alternate checkpoint is written"); + } + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var afterFiles = after.ReadFiles(afterCheckpoint); + + Assert.Multiple(() => { + Assert.That(afterCheckpoint.Lcn, Is.EqualTo(beforeCheckpoint.Lcn), + "a failed publication must not change which checkpoint is active"); + Assert.That(afterCheckpoint.Clock, Is.EqualTo(beforeCheckpoint.Clock)); + Assert.That(afterCheckpoint.Roots, Is.EqualTo(beforeCheckpoint.Roots)); + Assert.That(after.CountRefcountRows(afterCheckpoint), Is.Zero, + "no Block Refcount row may become live through an aborted clone"); + Assert.That(afterFiles["alpha.bin"].Clusters, Is.EqualTo(beforeFiles["alpha.bin"].Clusters)); + Assert.That(afterFiles["bravo.bin"].Clusters, Is.EqualTo(beforeFiles["bravo.bin"].Clusters)); + Assert.That(after.ReadFileContent(afterFiles["alpha.bin"]), Is.EqualTo(sourceBytes)); + Assert.That(after.ReadFileContent(afterFiles["bravo.bin"]), Is.EqualTo(destinationBytes)); + Assert.That(after.ReadAllocatedClusters(afterCheckpoint), Is.EquivalentTo(beforeAllocated), + "the destination's allocation must not be reclaimed before the clone is durable"); + }); + } + + [Test, Category("HappyPath")] + public void WritingToOneCloneAfterwards_LeavesTheOtherUntouched() { + var sourceBytes = Pattern(0x51, 2 * ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", sourceBytes) + .WithFile("bravo.bin", Pattern(0xB3, 2 * ClusterSize)) + .Build(); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var cloned = new RefsImageProbe(image); + var shared = cloned.ReadFiles(cloned.ActiveCheckpoint())["alpha.bin"].Clusters; + var replacement = Pattern(0xE7, 2 * ClusterSize); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineModifier.Add(stream, [ArchiveInputInfo.InMemory("bravo.bin", replacement)]); + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var afterFiles = after.ReadFiles(afterCheckpoint); + var refcounts = after.ReadRefcounts(afterCheckpoint); + var allocated = after.ReadAllocatedClusters(afterCheckpoint); + + Assert.Multiple(() => { + Assert.That(afterFiles["alpha.bin"].Clusters, Is.EqualTo(shared), + "the untouched clone must keep its extents"); + Assert.That(after.ReadFileContent(afterFiles["alpha.bin"]), Is.EqualTo(sourceBytes), + "writing to one clone changed the other's bytes"); + Assert.That(afterFiles["bravo.bin"].Clusters, Is.Not.EqualTo(shared), + "the written clone must land on fresh clusters, not over the shared ones"); + Assert.That(after.ReadFileContent(afterFiles["bravo.bin"]), Is.EqualTo(replacement)); + + foreach (var cluster in shared) { + Assert.That(allocated, Does.Contain(cluster), + $"Shared PLCN 0x{cluster:X} was freed while a stream still references it."); + var count = refcounts.TryGetValue(cluster, out var word) ? word.Count : (ushort)0; + Assert.That(count, Is.LessThanOrEqualTo(1), + $"PLCN 0x{cluster:X} still claims more than one owner after the second stream moved away."); + } + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesUnequalLogicalSizes() { + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", Pattern(0x61, ClusterSize)) + .WithFile("bravo.bin", Pattern(0xC1, 2 * ClusterSize)) + .Build(); + var untouched = (byte[])image.Clone(); + + using var stream = new MemoryStream(image, writable: true); + var error = Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin")); + Assert.Multiple(() => { + Assert.That(error!.Message, Does.Contain("equal source/destination logical sizes")); + Assert.That(image, Is.EqualTo(untouched), "a refused clone must not touch a single byte"); + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesNonClusterAlignedFiles() { + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", Pattern(0x71, ClusterSize - 17)) + .WithFile("bravo.bin", Pattern(0xD1, ClusterSize - 17)) + .Build(); + var untouched = (byte[])image.Clone(); + + using var stream = new MemoryStream(image, writable: true); + var error = Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin")); + Assert.Multiple(() => { + Assert.That(error!.Message, Does.Contain("cluster-aligned logical size")); + Assert.That(image, Is.EqualTo(untouched)); + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesResidentStreams() { + var content = Pattern(0x81, 48); + var image = new RefsSyntheticVolume() + .WithResidentFile("alpha.bin", content) + .WithFile("bravo.bin", Pattern(0xE1, ClusterSize)) + .Build(); + var untouched = (byte[])image.Clone(); + + using var stream = new MemoryStream(image, writable: true); + var error = Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin")); + Assert.Multiple(() => { + Assert.That(error!.Message, Does.Contain("extent-backed")); + Assert.That(image, Is.EqualTo(untouched)); + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesEmptyStreams() { + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", []) + .WithFile("bravo.bin", []) + .Build(); + var untouched = (byte[])image.Clone(); + + using var stream = new MemoryStream(image, writable: true); + Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin")); + Assert.That(image, Is.EqualTo(untouched)); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesAlreadySharedSourceClusters() { + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", Pattern(0x91, ClusterSize)) + .WithFile("bravo.bin", Pattern(0xA1, ClusterSize)) + .WithFile("carol.bin", Pattern(0xB1, ClusterSize)) + .Build(); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var afterFirst = (byte[])image.Clone(); + using var second = new MemoryStream(image, writable: true); + var error = Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(second, "alpha.bin", "carol.bin")); + Assert.Multiple(() => { + Assert.That(error!.Message, Does.Contain("already shared")); + Assert.That(image, Is.EqualTo(afterFirst), + "a refused repeat clone must not half-apply over the first one"); + }); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesTheSameFileAsSourceAndDestination() { + var image = new RefsSyntheticVolume().WithFile("alpha.bin", Pattern(0xA5, ClusterSize)).Build(); + using var stream = new MemoryStream(image, writable: true); + Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "alpha.bin")); + } + + [Test, Category("ErrorHandling")] + public void Clone_RefusesAnUnknownDestination() { + var image = new RefsSyntheticVolume().WithFile("alpha.bin", Pattern(0xA6, ClusterSize)).Build(); + using var stream = new MemoryStream(image, writable: true); + Assert.Throws( + () => RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "ghost.bin")); + } + + [Test, Category("EdgeCase")] + public void Clone_OfAFragmentedSourceCarriesEveryRunToTheDestination() { + var sourceBytes = Pattern(0xB7, 4 * ClusterSize); + var image = new RefsSyntheticVolume() + .WithFile("alpha.bin", sourceBytes, fragments: 3) + .WithFile("bravo.bin", Pattern(0xC7, 4 * ClusterSize)) + .Build(); + + var before = new RefsImageProbe(image); + var sourceView = before.ReadFiles(before.ActiveCheckpoint())["alpha.bin"]; + Assume.That(sourceView.Extents, Has.Count.EqualTo(3), "the fixture must fragment the source"); + + using (var stream = new MemoryStream(image, writable: true)) + RefsOfflineBlockCloner.CloneWholeFile(stream, "alpha.bin", "bravo.bin"); + + var after = new RefsImageProbe(image); + var afterCheckpoint = after.ActiveCheckpoint(); + var afterFiles = after.ReadFiles(afterCheckpoint); + var refcounts = after.ReadRefcounts(afterCheckpoint); + + Assert.Multiple(() => { + Assert.That(afterFiles["bravo.bin"].Extents, Has.Count.EqualTo(3), + "a fragmented clone must reproduce every run, not collapse or drop one"); + Assert.That(afterFiles["bravo.bin"].Clusters, Is.EqualTo(sourceView.Clusters)); + Assert.That(afterFiles["bravo.bin"].Extents.Select(e => (e.Vcn, e.Lcn, e.Run)), + Is.EqualTo(sourceView.Extents.Select(e => (e.Vcn, e.Lcn, e.Run)))); + foreach (var cluster in sourceView.Clusters) + Assert.That(refcounts[cluster].Count, Is.EqualTo(2)); + Assert.That(after.ReadFileContent(afterFiles["bravo.bin"]), Is.EqualTo(sourceBytes)); + Assert.That(after.ReadFileContent(afterFiles["alpha.bin"]), Is.EqualTo(sourceBytes)); + }); + } + + /// A memory image whose writes fail inside one byte range, to + /// simulate losing the medium at the exact moment of publication. + private sealed class WriteBarrierStream(byte[] buffer, long failFrom, long failTo) + : MemoryStream(buffer, writable: true) { + public bool Tripped { get; private set; } + + public override void Write(byte[] buffer, int offset, int count) { + this.Guard(count); + base.Write(buffer, offset, count); + } + + public override void Write(ReadOnlySpan buffer) { + this.Guard(buffer.Length); + base.Write(buffer); + } + + private void Guard(int count) { + var start = this.Position; + if (start + count <= failFrom || start >= failTo) return; + this.Tripped = true; + throw new IOException("Simulated media failure while publishing the alternate ReFS checkpoint."); + } + } +} diff --git a/Compression.Tests/Refs/RefsImageProbe.cs b/Compression.Tests/Refs/RefsImageProbe.cs new file mode 100644 index 000000000..d4066dd16 --- /dev/null +++ b/Compression.Tests/Refs/RefsImageProbe.cs @@ -0,0 +1,390 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; + +namespace Compression.Tests.Refs; + +/// +/// Independent on-disk decoder for ReFS images, written straight from the +/// structure definitions and deliberately sharing no code with the package's +/// reader, writer or CoW engine. +/// +/// Its whole purpose is to be a second pair of eyes: assertions that go through +/// RefsMetadataReader only prove that our reader agrees with our writer, +/// which is exactly how a shared defect survives. Everything this type returns +/// is decoded from raw bytes — checkpoint clocks, root descriptors, B+ rows, +/// Block Refcount words, allocator bitmaps and extent tables. +/// +internal sealed class RefsImageProbe { + public const int SuperblockCluster = 0x1E; + + private readonly byte[] _image; + + public RefsImageProbe(byte[] image) { + this._image = image; + if (!image.AsSpan(3, 4).SequenceEqual("ReFS"u8)) throw new InvalidDataException("Not a ReFS image."); + var sectorSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(0x20, 4)); + var sectorsPerCluster = (int)BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(0x24, 4)); + this.ClusterSize = sectorSize * sectorsPerCluster; + this.PageSize = this.ClusterSize <= 4096 ? 16 * 1024 : 64 * 1024; + this.ClustersPerPage = this.PageSize / this.ClusterSize; + this.LoadBootstrap(); + this.LoadContainers(); + } + + public int ClusterSize { get; } + public int PageSize { get; } + public int ClustersPerPage { get; } + public int PageReferenceSize { get; private set; } + public IReadOnlyList CheckpointLcns { get; private set; } = []; + public ulong ActiveCheckpointLcn { get; private set; } + public ulong ActiveCheckpointClock { get; private set; } + + private readonly Dictionary _containerBases = []; + private uint _clustersPerContainer; + private int _containerShift; + + // ── bootstrap ────────────────────────────────────────────────────────────── + + private void LoadBootstrap() { + var supb = this.Cluster(SuperblockCluster); + if (!supb.AsSpan(0, 4).SequenceEqual("SUPB"u8)) + throw new InvalidDataException("Probe found no SUPB at cluster 0x1E."); + var listOffset = (int)BinaryPrimitives.ReadUInt32LittleEndian(supb.AsSpan(0x70, 4)); + var listCount = (int)BinaryPrimitives.ReadUInt32LittleEndian(supb.AsSpan(0x74, 4)); + var slots = new List(listCount); + for (var i = 0; i < listCount; ++i) { + var lcn = BinaryPrimitives.ReadUInt64LittleEndian(supb.AsSpan(listOffset + i * 8, 8)); + if (lcn is not (0 or ulong.MaxValue)) slots.Add(lcn); + } + this.CheckpointLcns = slots; + + ulong bestLcn = 0; + ulong bestClock = 0; + var found = false; + foreach (var lcn in slots) { + var page = this.Page(lcn); + if (page == null || !page.AsSpan(0, 4).SequenceEqual("CHKP"u8)) continue; + var clock = BinaryPrimitives.ReadUInt64LittleEndian(page.AsSpan(0x60, 8)); + if (found && clock <= bestClock) continue; + bestLcn = lcn; + bestClock = clock; + found = true; + } + if (!found) throw new InvalidDataException("Probe found no readable CHKP."); + this.ActiveCheckpointLcn = bestLcn; + this.ActiveCheckpointClock = bestClock; + this.PageReferenceSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(this.Page(bestLcn)!.AsSpan(0x5C, 4)); + } + + /// Clock and the thirteen root page addresses of one checkpoint slot. + internal sealed record CheckpointView(ulong Lcn, ulong Clock, IReadOnlyList> Roots); + + public CheckpointView ReadCheckpoint(ulong lcn) { + var page = this.Page(lcn) ?? throw new InvalidDataException($"Checkpoint 0x{lcn:X} is unreadable."); + if (!page.AsSpan(0, 4).SequenceEqual("CHKP"u8)) + throw new InvalidDataException($"Cluster 0x{lcn:X} does not contain CHKP."); + var clock = BinaryPrimitives.ReadUInt64LittleEndian(page.AsSpan(0x60, 8)); + var referenceSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(0x5C, 4)); + var flags = BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(0x78, 4)); + var listBase = (flags & 0x0200) != 0 + ? (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(0x94, 4)) + : 0x94; + + var roots = new List>(13); + for (var i = 0; i < 13; ++i) { + var descriptor = (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(listBase + i * 4, 4)); + if (descriptor <= 0 || descriptor + referenceSize > page.Length) { + roots.Add([]); + continue; + } + roots.Add(ParseReference(page.AsSpan(descriptor, referenceSize))); + } + return new CheckpointView(lcn, clock, roots); + } + + public CheckpointView ActiveCheckpoint() => this.ReadCheckpoint(this.ActiveCheckpointLcn); + + private static IReadOnlyList ParseReference(ReadOnlySpan reference) { + var lcns = new List(4); + for (var i = 0; i < 4 && i * 8 + 8 <= reference.Length; ++i) { + var lcn = BinaryPrimitives.ReadUInt64LittleEndian(reference.Slice(i * 8, 8)); + if (lcn is not (0 or ulong.MaxValue)) lcns.Add(lcn); + } + return lcns; + } + + // ── container translation ────────────────────────────────────────────────── + + private void LoadContainers() { + var active = this.ActiveCheckpoint(); + foreach (var rootIndex in new[] { 7, 8 }) { + var root = active.Roots[rootIndex]; + if (root.Count == 0) continue; + var bases = new Dictionary(); + uint cpc = 0; + foreach (var (key, value) in this.WalkPhysicalTree(root)) { + if (key.Length < 16 || value.Length < 0x98) continue; + var id = BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(0, 8)); + var rowCpc = BinaryPrimitives.ReadUInt32LittleEndian(value.AsSpan(0x18, 4)); + if (rowCpc == 0) continue; + if (cpc == 0) cpc = rowCpc; + if (rowCpc != cpc) { bases.Clear(); break; } + bases[id] = BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(value.Length - 16, 8)); + } + if (bases.Count == 0 || cpc == 0 || (cpc & (cpc - 1)) != 0) continue; + foreach (var item in bases) this._containerBases[item.Key] = item.Value; + this._clustersPerContainer = cpc; + this._containerShift = 0; + while ((1u << this._containerShift) < cpc) ++this._containerShift; + return; + } + throw new InvalidDataException("Probe could not decode a Container Table."); + } + + public ulong ToPhysical(ulong virtualLcn) { + var container = virtualLcn >> this._containerShift; + var within = virtualLcn & (this._clustersPerContainer - 1UL); + if (!this._containerBases.TryGetValue(container, out var physicalBase)) + throw new InvalidDataException($"Probe cannot map virtual container {container}."); + return physicalBase + within; + } + + // ── B+ walking ───────────────────────────────────────────────────────────── + + public IReadOnlyList<(byte[] Key, byte[] Value)> WalkVirtualTree(IReadOnlyList rootLcns) + => this.Walk(rootLcns.Select(this.ToPhysical).ToArray(), virtualChildren: true); + + public IReadOnlyList<(byte[] Key, byte[] Value)> WalkPhysicalTree(IReadOnlyList rootLcns) + => this.Walk(rootLcns, virtualChildren: false); + + private IReadOnlyList<(byte[] Key, byte[] Value)> Walk( + IReadOnlyList physicalLcns, + bool virtualChildren) { + var result = new List<(byte[], byte[])>(); + var seen = new HashSet(); + var pending = new Stack>(); + pending.Push(physicalLcns); + + while (pending.Count > 0) { + var slots = pending.Pop(); + if (slots.Count == 0 || !seen.Add(slots[0])) continue; + var page = this.PageFromSlots(slots); + if (page == null || !page.AsSpan(0, 4).SequenceEqual("MSB+"u8)) continue; + + var nodeOffset = 0x50 + (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(0x50, 4)); + var isInner = (BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(nodeOffset + 0x0C, 4)) & 0x100) != 0; + var indexStart = nodeOffset + (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(nodeOffset + 0x10, 4)); + var indexEnd = nodeOffset + (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(nodeOffset + 0x20, 4)); + + for (var entry = indexStart; entry < indexEnd; entry += 4) { + var rowOffset = nodeOffset + (int)(BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(entry, 4)) & 0xFFFF); + var rowSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(rowOffset, 4)); + if (rowSize < 16 || rowOffset + rowSize > page.Length) continue; + var keyOffset = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(rowOffset + 4, 2)); + var keyLength = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(rowOffset + 6, 2)); + var valueOffset = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(rowOffset + 10, 2)); + var valueLength = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(rowOffset + 12, 2)); + if (keyOffset + keyLength > rowSize || valueOffset + valueLength > rowSize) continue; + var key = page.AsSpan(rowOffset + keyOffset, keyLength).ToArray(); + var value = page.AsSpan(rowOffset + valueOffset, valueLength).ToArray(); + + if (!isInner) { + result.Add((key, value)); + continue; + } + if (value.Length < 32) continue; + var child = ParseReference(value); + if (child.Count == 0) continue; + pending.Push(virtualChildren ? child.Select(this.ToPhysical).ToArray() : child); + } + } + return result; + } + + // ── decoded views ────────────────────────────────────────────────────────── + + /// One decoded Block Refcount word: the 14-bit count and its two flag bits. + internal readonly record struct RefcountWord(ushort Count, bool DedupMetadata, bool DedupManaged); + + /// + /// Decodes every tracked Block Refcount slot of root #6 straight from the + /// 0x820-byte rows. Only slots inside an existing row appear; absence is the + /// sparse "one ordinary owner" state. + /// + public IReadOnlyDictionary ReadRefcounts(CheckpointView checkpoint) { + var result = new Dictionary(); + foreach (var (key, value) in this.WalkVirtualTree(checkpoint.Roots[6])) { + if (key.Length < 16 || value.Length < 0x820) continue; + var start = BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(0, 8)); + if (BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(8, 8)) != 0x400) continue; + for (var i = 0; i < 0x400; ++i) { + var raw = BinaryPrimitives.ReadUInt16LittleEndian(value.AsSpan(0x1C + i * 2, 2)); + if (raw == 0) continue; + result[start + (ulong)i] = new RefcountWord( + (ushort)(raw & 0x3FFF), + (raw & 0x4000) != 0, + (raw & 0x8000) != 0); + } + } + return result; + } + + /// Sum of the low 14-bit counts each Block Refcount row advertises at +0x18. + public IReadOnlyDictionary ReadRefcountTotals(CheckpointView checkpoint) { + var result = new Dictionary(); + foreach (var (key, value) in this.WalkVirtualTree(checkpoint.Roots[6])) { + if (key.Length < 16 || value.Length < 0x820) continue; + if (BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(8, 8)) != 0x400) continue; + var start = BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(0, 8)); + uint actual = 0; + for (var i = 0; i < 0x400; ++i) + actual += (uint)(BinaryPrimitives.ReadUInt16LittleEndian(value.AsSpan(0x1C + i * 2, 2)) & 0x3FFF); + result[start] = (BinaryPrimitives.ReadUInt32LittleEndian(value.AsSpan(0x18, 4)), actual); + } + return result; + } + + public int CountRefcountRows(CheckpointView checkpoint) + => this.WalkVirtualTree(checkpoint.Roots[6]) + .Count(row => row.Key.Length >= 16 + && BinaryPrimitives.ReadUInt64LittleEndian(row.Key.AsSpan(8, 8)) == 0x400); + + /// Allocation state of the Medium Allocator (root #1) as a physical-LCN set. + public HashSet ReadAllocatedClusters(CheckpointView checkpoint) { + var allocated = new HashSet(); + foreach (var (_, value) in this.WalkVirtualTree(checkpoint.Roots[1])) { + if (value.Length < 24) continue; + var start = BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(0x00, 8)); + var length = BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(0x08, 8)); + var flags = BinaryPrimitives.ReadUInt16LittleEndian(value.AsSpan(0x12, 2)); + if (length == 0 || length > 16384) continue; + for (ulong i = 0; i < length; ++i) { + var isAllocated = flags switch { + 0x01 when value.Length >= 0x18 + 2048 => (value[0x18 + (int)(i >> 3)] & (1 << (int)(i & 7))) != 0, + 0x02 => true, + 0x05 or 0x09 => false, + _ => throw new InvalidDataException($"Probe met an allocator row with flags 0x{flags:X4}."), + }; + if (isAllocated) allocated.Add(this.ToPhysical(start + i)); + } + } + return allocated; + } + + /// A file's on-disk stream layout, decoded from its type-0x40 backing row. + internal sealed record FileView( + string Name, + ulong FileId, + ulong Size, + ulong AllocatedSize, + IReadOnlyList<(uint Vcn, ulong Lcn, uint Run, uint Flags)> Extents) { + public IReadOnlyList Clusters { + get { + var result = new List(); + foreach (var extent in this.Extents.OrderBy(e => e.Vcn)) + for (uint i = 0; i < extent.Run; ++i) + result.Add(extent.Lcn + i); + return result; + } + } + } + + public IReadOnlyDictionary ReadFiles(CheckpointView checkpoint) { + var objects = new Dictionary>(); + foreach (var (key, value) in this.WalkVirtualTree(checkpoint.Roots[0])) { + if (key.Length < 16 || value.Length < 0x40) continue; + var oid = BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(8, 8)); + var reference = ParseReference(value.AsSpan(0x20)); + if (reference.Count > 0) objects[oid] = reference; + } + if (!objects.TryGetValue(RefsSyntheticVolume.RootDirectoryOid, out var directoryRoot)) + throw new InvalidDataException("Probe found no root directory object."); + + var rows = this.WalkVirtualTree(directoryRoot); + var backing = new Dictionary(); + foreach (var (key, value) in rows) { + if (key.Length < 16 || BinaryPrimitives.ReadUInt16LittleEndian(key.AsSpan(0, 2)) != 0x40) continue; + backing[BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(8, 8))] = value; + } + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var (key, value) in rows) { + if (key.Length < 4 || BinaryPrimitives.ReadUInt16LittleEndian(key.AsSpan(0, 2)) != 0x30) continue; + if (BinaryPrimitives.ReadUInt16LittleEndian(key.AsSpan(2, 2)) != 0x02) continue; + if (value.Length is < 0x48 or > 0x54) continue; + var name = Encoding.Unicode.GetString(key.AsSpan(4)).TrimEnd('\0'); + var fileId = BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(0x00, 8)); + if (!backing.TryGetValue(fileId, out var holder)) continue; + result[name] = new FileView( + name, + fileId, + BinaryPrimitives.ReadUInt64LittleEndian(holder.AsSpan(0x58, 8)), + BinaryPrimitives.ReadUInt64LittleEndian(holder.AsSpan(0x60, 8)), + this.DecodeExtents(holder)); + } + return result; + } + + private IReadOnlyList<(uint Vcn, ulong Lcn, uint Run, uint Flags)> DecodeExtents(byte[] holder) { + var result = new List<(uint, ulong, uint, uint)>(); + if (holder.Length < 0xB0) return result; + var recordSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(holder.AsSpan(0xA8, 4)); + if (recordSize <= 0 || 0xA8 + recordSize > holder.Length) return result; + var header = 0xAC; + var start = (int)BinaryPrimitives.ReadUInt32LittleEndian(holder.AsSpan(header + 0x00, 4)); + var count = (int)BinaryPrimitives.ReadUInt32LittleEndian(holder.AsSpan(header + 0x14, 4)); + var entries = header + start; + for (var i = 0; i < count && entries + (i + 1) * 24 <= holder.Length; ++i) { + var entry = holder.AsSpan(entries + i * 24, 24); + result.Add(( + BinaryPrimitives.ReadUInt32LittleEndian(entry[0x0C..0x10]), + this.ToPhysical(BinaryPrimitives.ReadUInt64LittleEndian(entry[0x00..0x08])), + BinaryPrimitives.ReadUInt32LittleEndian(entry[0x14..0x18]), + BinaryPrimitives.ReadUInt32LittleEndian(entry[0x08..0x0C]))); + } + return result; + } + + /// Reads a file's bytes by following its decoded extents in the raw image. + public byte[] ReadFileContent(FileView file) { + var result = new byte[file.Size]; + var written = 0; + foreach (var lcn in file.Clusters) { + var take = Math.Min(this.ClusterSize, result.Length - written); + if (take <= 0) break; + this._image.AsSpan((int)((long)lcn * this.ClusterSize), take).CopyTo(result.AsSpan(written)); + written += take; + } + return result; + } + + // ── raw access ───────────────────────────────────────────────────────────── + + public byte[] Cluster(ulong lcn) { + var offset = (long)lcn * this.ClusterSize; + if (offset < 0 || offset + this.ClusterSize > this._image.Length) + throw new InvalidDataException($"Cluster 0x{lcn:X} lies outside the image."); + return this._image.AsSpan((int)offset, this.ClusterSize).ToArray(); + } + + public byte[]? Page(ulong headLcn) { + var offset = (long)headLcn * this.ClusterSize; + if (offset < 0 || offset + this.PageSize > this._image.Length) return null; + return this._image.AsSpan((int)offset, this.PageSize).ToArray(); + } + + private byte[]? PageFromSlots(IReadOnlyList slots) { + var page = new byte[this.PageSize]; + var written = 0; + foreach (var lcn in slots) { + if (written >= page.Length) break; + var offset = (long)lcn * this.ClusterSize; + if (offset < 0 || offset + this.ClusterSize > this._image.Length) return null; + var take = Math.Min(this.ClusterSize, page.Length - written); + this._image.AsSpan((int)offset, take).CopyTo(page.AsSpan(written, take)); + written += take; + } + return written == page.Length ? page : null; + } +} diff --git a/Compression.Tests/Refs/RefsSyntheticVolume.cs b/Compression.Tests/Refs/RefsSyntheticVolume.cs new file mode 100644 index 000000000..c6c628352 --- /dev/null +++ b/Compression.Tests/Refs/RefsSyntheticVolume.cs @@ -0,0 +1,549 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; + +namespace Compression.Tests.Refs; + +/// One file to place into a synthetic ReFS volume. +/// Leaf name inside the root directory. +/// Stream bytes; the allocation is rounded up to a cluster. +/// Number of on-disk extents the stream is split into. +/// Emit a type-0x30/flags-0x01 holder without any decodable +/// extent table, which the namespace reader reports as a resident stream. +internal sealed record RefsSyntheticFile( + string Name, + byte[] Content, + int Fragments = 1, + bool Resident = false); + +/// +/// Builds a byte-exact, structurally valid unmounted ReFS 3.14 image. +/// +/// The package deliberately has no ReFS creator — its writer edits existing +/// volumes only — so the offline block-clone primitive had no image to run +/// against and therefore no executing test. This generator supplies one. It is +/// a test fixture, not a mkfs: it emits the smallest metadata graph that the +/// offline mutation stack needs (VBR, SUPB, two CHKP slots, Container Table, +/// Object Table, Schema Table, Medium Allocator, Block Refcount and a single +/// root-directory object), every tree being one leaf page so no B+ split, no +/// separator-key convention and no Upcase Table are involved. +/// +/// Two fixture choices are worth naming because they differ from a Windows +/// volume: the Container Table publishes one identity container, so VLCN == +/// PLCN throughout; and the root-directory table is stamped with the generic +/// attribute schema 0x120 rather than the filename schema 0x140, so key +/// ordering never needs the 128 KiB Upcase Table. Neither is reachable from +/// the behaviour under test — the cloner never splits a page and never +/// compares filename keys. +/// +internal sealed class RefsSyntheticVolume { + public const int SectorSize = 512; + public const int SectorsPerCluster = 8; + public const int ClusterSize = SectorSize * SectorsPerCluster; + public const int PageSize = 16 * 1024; + public const int ClustersPerPage = PageSize / ClusterSize; + public const int PageReferenceSize = 48; + public const uint ClustersPerContainer = 16384; + public const int TotalClusters = 1024; + public const ulong RootDirectoryOid = 0x600; + public const uint ExtentFlags = 0x180040; + + // Fixed placement. Every metadata page occupies ClustersPerPage clusters. + public const ulong VbrCluster = 0; + public const ulong SuperblockCluster = 0x1E; + public const ulong CheckpointACluster = 36; + public const ulong CheckpointBCluster = 40; + public const ulong ContainerRootCluster = 44; + public const ulong ObjectRootCluster = 48; + public const ulong SchemaRootCluster = 52; + public const ulong MediumAllocatorRootCluster = 56; + public const ulong BlockRefcountRootCluster = 60; + public const ulong DirectoryRootCluster = 64; + public const ulong FirstDataCluster = 68; + + private const int NodeOffset = 0x80; + private const int DataStart = 0xA8; + private const int IndexEnd = PageSize; + private const int SchemaDescriptorOffset = 0x54; + + private readonly List _files = []; + private readonly List _seededRefcountRows = []; + + /// Layout of a built volume: where every file's clusters landed. + internal sealed record Layout( + IReadOnlyDictionary> FileClusters, + IReadOnlyDictionary FileIds, + ulong FirstFreeCluster); + + public Layout? BuiltLayout { get; private set; } + + public RefsSyntheticVolume WithFile(string name, byte[] content, int fragments = 1) { + this._files.Add(new RefsSyntheticFile(name, content, fragments)); + return this; + } + + public RefsSyntheticVolume WithResidentFile(string name, byte[] content) { + this._files.Add(new RefsSyntheticFile(name, content, 1, Resident: true)); + return this; + } + + /// + /// Pre-creates an all-zero Block Refcount row covering . + /// Without it root #6 is empty and the first clone must materialise the row; + /// with it the row exists but the target slot is zero, which is the second + /// first-clone case the codec distinguishes. + /// + public RefsSyntheticVolume WithEmptyRefcountRow(ulong startVirtualLcn = 0) { + this._seededRefcountRows.Add(startVirtualLcn); + return this; + } + + public byte[] Build() { + var image = new byte[(long)TotalClusters * ClusterSize]; + + // ── data placement ─────────────────────────────────────────────────────── + var fileClusters = new Dictionary>(); + var fileExtents = new Dictionary>(); + var fileIds = new Dictionary(); + var cursor = FirstDataCluster; + ulong nextFileId = 0x1000; + + foreach (var file in this._files) { + fileIds[file.Name] = nextFileId++; + if (file.Resident) { + fileClusters[file.Name] = []; + fileExtents[file.Name] = []; + continue; + } + + var clusterCount = (file.Content.Length + ClusterSize - 1) / ClusterSize; + var clusters = new List(clusterCount); + var extents = new List<(ulong Lcn, uint Vcn, uint Run)>(); + if (clusterCount > 0) { + var fragments = Math.Max(1, Math.Min(file.Fragments, clusterCount)); + var perFragment = clusterCount / fragments; + var remainder = clusterCount % fragments; + uint vcn = 0; + for (var f = 0; f < fragments; ++f) { + var run = perFragment + (f < remainder ? 1 : 0); + extents.Add((cursor, vcn, (uint)run)); + for (var i = 0; i < run; ++i) clusters.Add(cursor + (ulong)i); + vcn += (uint)run; + cursor += (ulong)run; + if (f + 1 < fragments) ++cursor; // deliberate hole so the runs stay separate + } + } + fileClusters[file.Name] = clusters; + fileExtents[file.Name] = extents; + + var written = 0; + foreach (var lcn in clusters) { + var take = Math.Min(ClusterSize, file.Content.Length - written); + if (take <= 0) break; + file.Content.AsSpan(written, take).CopyTo(image.AsSpan((int)(lcn * ClusterSize), take)); + written += take; + } + } + + var firstFree = Math.Max(cursor, FirstDataCluster); + this.BuiltLayout = new Layout(fileClusters, fileIds, firstFree); + + // ── bootstrap structures ───────────────────────────────────────────────── + WriteVbr(image); + WriteSuperblock(image); + WriteCheckpoint(image, CheckpointACluster, clock: 2); + WriteCheckpoint(image, CheckpointBCluster, clock: 1); + + // ── metadata trees ─────────────────────────────────────────────────────── + WritePage(image, ContainerRootCluster, tableId: 0x0B, schemaId: 0xE0C0, physical: true, + rows: [new Row(ContainerKey(), ContainerValue())]); + + WritePage(image, ObjectRootCluster, tableId: 0x02, schemaId: 0xE030, physical: false, + rows: [new Row(ObjectKey(RootDirectoryOid), ObjectValue(DirectoryRootCluster))]); + + WritePage(image, SchemaRootCluster, tableId: 0x03, schemaId: 0xE060, physical: false, + rows: SchemaRows()); + + WritePage(image, MediumAllocatorRootCluster, tableId: 0x21, schemaId: 0xE010, physical: false, + rows: [new Row(AllocatorKey(), AllocatorValue(firstFree))]); + + WritePage(image, BlockRefcountRootCluster, tableId: 0x0F, schemaId: 0xE0B0, physical: false, + rows: this._seededRefcountRows + .Distinct() + .OrderBy(start => start) + .Select(start => new Row(RefcountKey(start), RefcountValue(start))) + .ToList()); + + WritePage(image, DirectoryRootCluster, tableId: 0x300, schemaId: 0x120, physical: false, + rows: this.DirectoryRows(fileIds, fileExtents)); + + return image; + } + + // ── bootstrap ────────────────────────────────────────────────────────────── + + private static void WriteVbr(byte[] image) { + var vbr = image.AsSpan(0, 512); + Encoding.ASCII.GetBytes("ReFS").CopyTo(vbr[3..]); + Encoding.ASCII.GetBytes("FSRS").CopyTo(vbr[0x10..]); + BinaryPrimitives.WriteUInt16LittleEndian(vbr.Slice(0x14, 2), 0x200); + BinaryPrimitives.WriteUInt64LittleEndian(vbr.Slice(0x18, 8), (ulong)TotalClusters * SectorsPerCluster); + BinaryPrimitives.WriteUInt32LittleEndian(vbr.Slice(0x20, 4), SectorSize); + BinaryPrimitives.WriteUInt32LittleEndian(vbr.Slice(0x24, 4), SectorsPerCluster); + vbr[0x28] = 3; + vbr[0x29] = 14; + BinaryPrimitives.WriteUInt16LittleEndian(vbr.Slice(0x2A, 2), 2); + BinaryPrimitives.WriteUInt32LittleEndian(vbr.Slice(0x2C, 4), 0x66); + BinaryPrimitives.WriteUInt64LittleEndian(vbr.Slice(0x38, 8), 0x0123456789ABCDEFUL); + BinaryPrimitives.WriteUInt64LittleEndian(vbr.Slice(0x40, 8), (ulong)ClustersPerContainer * ClusterSize); + BinaryPrimitives.WriteUInt16LittleEndian(vbr.Slice(0x16, 2), VbrChecksum(vbr)); + } + + internal static ushort VbrChecksum(ReadOnlySpan vbr) { + ushort sum = 0; + for (var i = 3; i < 512; ++i) { + if (i is 0x16 or 0x17) continue; + sum = (ushort)((sum >> 1) | (sum << 15)); + sum = unchecked((ushort)(sum + vbr[i])); + } + return sum; + } + + private static void WriteSuperblock(byte[] image) { + var supb = image.AsSpan((int)(SuperblockCluster * ClusterSize), ClusterSize); + Encoding.ASCII.GetBytes("SUPB").CopyTo(supb); + BinaryPrimitives.WriteUInt64LittleEndian(supb.Slice(0x68, 8), 1); + BinaryPrimitives.WriteUInt32LittleEndian(supb.Slice(0x70, 4), 0x80); + BinaryPrimitives.WriteUInt32LittleEndian(supb.Slice(0x74, 4), 2); + BinaryPrimitives.WriteUInt32LittleEndian(supb.Slice(0x78, 4), 0x100); + BinaryPrimitives.WriteUInt32LittleEndian(supb.Slice(0x7C, 4), PageReferenceSize); + BinaryPrimitives.WriteUInt64LittleEndian(supb.Slice(0x80, 8), CheckpointACluster); + BinaryPrimitives.WriteUInt64LittleEndian(supb.Slice(0x88, 8), CheckpointBCluster); + // Self-checksum descriptor: type 0 / length 0, so every Refresh* call is a + // no-op. Nothing on the ReFS read path verifies a digest. + } + + private static void WriteCheckpoint(byte[] image, ulong headCluster, ulong clock) { + var page = new byte[PageSize]; + Encoding.ASCII.GetBytes("CHKP").CopyTo(page.AsSpan()); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x10, 8), clock); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x20, 8), headCluster); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x58, 4), 0x380); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x5C, 4), PageReferenceSize); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x60, 8), clock); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x68, 8), clock); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x78, 4), 0x0002); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x90, 4), 13); + for (var i = 0; i < 13; ++i) + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x94 + i * 4, 4), (uint)(0x100 + i * PageReferenceSize)); + + StampRoot(page, 0, ObjectRootCluster); + StampRoot(page, 1, MediumAllocatorRootCluster); + StampRoot(page, 3, SchemaRootCluster); + StampRoot(page, 6, BlockRefcountRootCluster); + StampRoot(page, 7, ContainerRootCluster); + + WriteClusters(image, headCluster, page); + } + + private static void StampRoot(byte[] checkpoint, int rootIndex, ulong headCluster) { + var descriptor = 0x100 + rootIndex * PageReferenceSize; + for (var i = 0; i < ClustersPerPage; ++i) + BinaryPrimitives.WriteUInt64LittleEndian( + checkpoint.AsSpan(descriptor + i * 8, 8), + headCluster + (ulong)i); + // reference[0x22] = checksum type 0, reference[0x24] = digest length 0. + } + + private static void WriteClusters(byte[] image, ulong headCluster, ReadOnlySpan page) { + for (var i = 0; i < page.Length; i += ClusterSize) + page.Slice(i, Math.Min(ClusterSize, page.Length - i)) + .CopyTo(image.AsSpan((int)(headCluster * ClusterSize) + i)); + } + + // ── B+ leaf pages ────────────────────────────────────────────────────────── + + internal sealed record Row(byte[] Key, byte[] Value); + + private static void WritePage( + byte[] image, + ulong headCluster, + ulong tableId, + ushort schemaId, + bool physical, + IReadOnlyList rows) { + var page = new byte[PageSize]; + Encoding.ASCII.GetBytes("MSB+").CopyTo(page.AsSpan()); + for (var i = 0; i < ClustersPerPage; ++i) + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x20 + i * 8, 8), headCluster + (ulong)i); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(0x48, 8), tableId); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(0x50, 4), NodeOffset - 0x50); + + // IndexRoot descriptor: RefsCowBTree.FindRootSchema scans [0x50, nodeOffset) + // for a 0x28-byte descriptor and reads the schema id from +0x0C. + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(SchemaDescriptorOffset, 4), 0x28); + BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(SchemaDescriptorOffset + 0x0C, 2), schemaId); + BinaryPrimitives.WriteUInt64LittleEndian(page.AsSpan(SchemaDescriptorOffset + 0x20, 8), (ulong)rows.Count); + + var serialized = rows.Select(SerializeRow).ToArray(); + var rowBytes = serialized.Sum(r => r.Length); + var indexStart = IndexEnd - rows.Count * 4; + if (DataStart + rowBytes > indexStart) + throw new InvalidOperationException("Synthetic ReFS page overflows; the fixture needs a smaller tree."); + + var offset = DataStart; + for (var i = 0; i < serialized.Length; ++i) { + serialized[i].CopyTo(page, offset); + BinaryPrimitives.WriteUInt32LittleEndian( + page.AsSpan(indexStart + i * 4, 4), + 0xFFFF0000U | (uint)(offset - NodeOffset)); + offset += serialized[i].Length; + } + + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x00, 4), DataStart - NodeOffset); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x04, 4), (uint)(offset - NodeOffset)); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x08, 4), (uint)(indexStart - offset)); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x0C, 4), 0); // leaf + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x10, 4), (uint)(indexStart - NodeOffset)); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x14, 4), (uint)rows.Count); + BinaryPrimitives.WriteUInt32LittleEndian(page.AsSpan(NodeOffset + 0x20, 4), IndexEnd - NodeOffset); + + WriteClusters(image, headCluster, page); + } + + private static byte[] SerializeRow(Row row) { + const int keyOffset = 16; + var valueOffset = Align8(keyOffset + row.Key.Length); + var rowSize = Align8(valueOffset + row.Value.Length); + var result = new byte[rowSize]; + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(0, 4), (uint)rowSize); + BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(4, 2), keyOffset); + BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(6, 2), (ushort)row.Key.Length); + BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(10, 2), (ushort)valueOffset); + BinaryPrimitives.WriteUInt16LittleEndian(result.AsSpan(12, 2), (ushort)row.Value.Length); + row.Key.CopyTo(result, keyOffset); + row.Value.CopyTo(result, valueOffset); + return result; + } + + private static int Align8(int value) => (value + 7) & ~7; + + // ── row builders ─────────────────────────────────────────────────────────── + + private static byte[] ContainerKey() => new byte[16]; + + private static byte[] ContainerValue() { + var value = new byte[0x98]; + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x18, 4), ClustersPerContainer); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(value.Length - 16, 8), 0UL); + return value; + } + + private static byte[] ObjectKey(ulong oid) { + var key = new byte[16]; + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), oid); + return key; + } + + private static byte[] ObjectValue(ulong rootHeadCluster) { + var value = new byte[0x20 + PageReferenceSize]; + for (var i = 0; i < ClustersPerPage; ++i) + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x20 + i * 8, 8), rootHeadCluster + (ulong)i); + return value; + } + + private static readonly uint[] SchemaIds = [ + 0x110, 0x120, 0x130, 0x140, + 0xE010, 0xE030, 0xE040, 0xE060, 0xE080, 0xE090, 0xE0B0, 0xE0C0, + ]; + + private static List SchemaRows() { + var rows = new List(SchemaIds.Length); + foreach (var schemaId in SchemaIds.OrderBy(id => id)) { + var key = new byte[8]; + BinaryPrimitives.WriteUInt32LittleEndian(key.AsSpan(0, 4), schemaId); + var value = new byte[0x50]; + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x00, 4), 0x50); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x04, 4), 0x18); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x18, 4), 0x38); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x1C, 4), 1); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x24, 4), schemaId); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x48, 4), 1); + rows.Add(new Row(key, value)); + } + return rows; + } + + private static byte[] AllocatorKey() { + var key = new byte[16]; + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(0, 8), 0UL); + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), TotalClusters); + return key; + } + + /// + /// One bitmap Medium Allocator row covering the whole volume. Everything below + /// is metadata or file data; the rest is the pool + /// the CoW page store reserves its replacement pages from. + /// + private static byte[] AllocatorValue(ulong firstFree) { + var value = new byte[0x18 + 2048]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x00, 8), 0UL); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x08, 8), TotalClusters); + + var used = 0; + for (ulong lcn = 0; lcn < firstFree; ++lcn) { + value[0x18 + (int)(lcn >> 3)] |= (byte)(1 << (int)(lcn & 7)); + ++used; + } + + BinaryPrimitives.WriteUInt16LittleEndian(value.AsSpan(0x10, 2), (ushort)(TotalClusters - used)); + BinaryPrimitives.WriteUInt16LittleEndian(value.AsSpan(0x12, 2), 0x01); + BinaryPrimitives.WriteUInt16LittleEndian(value.AsSpan(0x14, 2), 0x0218); + BinaryPrimitives.WriteUInt16LittleEndian(value.AsSpan(0x16, 2), (ushort)used); + return value; + } + + private static byte[] RefcountKey(ulong startVirtualLcn) { + var key = new byte[16]; + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(0, 8), startVirtualLcn); + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), 0x400); + return key; + } + + private static byte[] RefcountValue(ulong startVirtualLcn) { + var value = new byte[0x820]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x00, 8), startVirtualLcn); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x08, 8), 0x400); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x10, 8), 1UL); + return value; + } + + private List DirectoryRows( + IReadOnlyDictionary fileIds, + IReadOnlyDictionary> fileExtents) { + var rows = new List(); + foreach (var file in this._files) { + var fileId = fileIds[file.Name]; + var size = (ulong)file.Content.Length; + var allocated = (ulong)(((file.Content.Length + ClusterSize - 1) / ClusterSize) * (long)ClusterSize); + + if (file.Resident) { + rows.Add(new Row(NameKey(file.Name, 0x01), ResidentHolderValue(size))); + continue; + } + + rows.Add(new Row(NameKey(file.Name, 0x02), FileRecordValue(fileId, size, allocated))); + rows.Add(new Row(BackingKey(fileId), BackingValue(size, allocated, fileExtents[file.Name]))); + } + rows.Sort((a, b) => CompareAttributeKey(a.Key, b.Key)); + return rows; + } + + private static byte[] NameKey(string name, ushort keyFlags) { + var encoded = Encoding.Unicode.GetBytes(name); + var key = new byte[4 + encoded.Length]; + BinaryPrimitives.WriteUInt16LittleEndian(key.AsSpan(0, 2), 0x30); + BinaryPrimitives.WriteUInt16LittleEndian(key.AsSpan(2, 2), keyFlags); + encoded.CopyTo(key, 4); + return key; + } + + private static byte[] FileRecordValue(ulong fileId, ulong size, ulong allocated) { + var value = new byte[0x48]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x00, 8), fileId); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x08, 8), RootDirectoryOid); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x18, 8), 0x01D0_0000_0000_0000UL); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x30, 8), allocated); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x38, 8), size); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x40, 4), 0x20); + return value; + } + + private static byte[] ResidentHolderValue(ulong size) { + var value = new byte[0x80]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x30, 8), 0x01D0_0000_0000_0000UL); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(0x48, 4), 0x20); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x58, 8), size); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x60, 8), 0UL); + return value; + } + + private static byte[] BackingKey(ulong fileId) { + var key = new byte[16]; + BinaryPrimitives.WriteUInt16LittleEndian(key.AsSpan(0, 2), 0x40); + BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), fileId); + return key; + } + + // Native 3.10+ extent-holder value. The sub-record starts at +0xA8, its extent + // table header at +0xAC and the 24-byte entries at +0xD0 — the layout the + // offline writer's TryRewriteNativeHolder recognises and rewrites in place. + public const int BackingSubRecordOffset = 0xA8; + public const int BackingTableHeaderOffset = 0xAC; + public const int BackingEntriesOffset = 0xD0; + + internal static byte[] BackingValue( + ulong size, + ulong allocated, + IReadOnlyList<(ulong Lcn, uint Vcn, uint Run)> extents) { + var length = BackingEntriesOffset + extents.Count * 24; + var value = new byte[Math.Max(length, 0x68)]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x58, 8), size); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x60, 8), allocated); + if (extents.Count == 0) return value; + + const int start = BackingEntriesOffset - BackingTableHeaderOffset; + BinaryPrimitives.WriteUInt32LittleEndian( + value.AsSpan(BackingSubRecordOffset, 4), (uint)(length - BackingSubRecordOffset)); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(BackingTableHeaderOffset + 0x00, 4), start); + BinaryPrimitives.WriteUInt32LittleEndian( + value.AsSpan(BackingTableHeaderOffset + 0x04, 4), (uint)(start + extents.Count * 24)); + BinaryPrimitives.WriteUInt32LittleEndian(value.AsSpan(BackingTableHeaderOffset + 0x0C, 4), 0xE00); + BinaryPrimitives.WriteUInt32LittleEndian( + value.AsSpan(BackingTableHeaderOffset + 0x14, 4), (uint)extents.Count); + + for (var i = 0; i < extents.Count; ++i) { + var entry = value.AsSpan(BackingEntriesOffset + i * 24, 24); + BinaryPrimitives.WriteUInt64LittleEndian(entry[0x00..0x08], extents[i].Lcn); + BinaryPrimitives.WriteUInt32LittleEndian(entry[0x08..0x0C], ExtentFlags); + BinaryPrimitives.WriteUInt32LittleEndian(entry[0x0C..0x10], extents[i].Vcn); + BinaryPrimitives.WriteUInt32LittleEndian(entry[0x14..0x18], extents[i].Run); + } + return value; + } + + /// + /// Replica of the key ordering ReFS uses for the generic attribute schemas: + /// u16 type, u16 flags, then little-endian u64 chunks, then a byte tail. The + /// fixture must emit rows in exactly this order or the CoW engine rejects the + /// tree before it mutates anything. + /// + internal static int CompareAttributeKey(ReadOnlySpan left, ReadOnlySpan right) { + if (left.Length >= 4 && right.Length >= 4) { + var type = BinaryPrimitives.ReadUInt16LittleEndian(left[..2]) + .CompareTo(BinaryPrimitives.ReadUInt16LittleEndian(right[..2])); + if (type != 0) return type; + var flags = BinaryPrimitives.ReadUInt16LittleEndian(left.Slice(2, 2)) + .CompareTo(BinaryPrimitives.ReadUInt16LittleEndian(right.Slice(2, 2))); + if (flags != 0) return flags; + } + + var cursor = 4; + while (cursor + 8 <= left.Length && cursor + 8 <= right.Length) { + var cmp = BinaryPrimitives.ReadUInt64LittleEndian(left.Slice(cursor, 8)) + .CompareTo(BinaryPrimitives.ReadUInt64LittleEndian(right.Slice(cursor, 8))); + if (cmp != 0) return cmp; + cursor += 8; + } + + var tailLeft = left[cursor..]; + var tailRight = right[cursor..]; + var count = Math.Min(tailLeft.Length, tailRight.Length); + for (var i = 0; i < count; ++i) { + var cmp = tailLeft[i].CompareTo(tailRight[i]); + if (cmp != 0) return cmp; + } + return tailLeft.Length.CompareTo(tailRight.Length); + } +} diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index 387b5e579..b160fab64 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -330,7 +330,7 @@ cwb recover raw.img --mode files --format Jpeg,Png - This is an image-manipulation toolkit, not a kernel filesystem driver. R/W does not imply concurrent mount semantics, crash-consistent journaling under arbitrary interruption, or a drop-in replacement for the OS driver. - Where **Notes** says a format's edits *rebuild* the volume, an add or remove costs the whole image rather than the bytes that changed, and the block placement of untouched files may move. - **bcachefs** mutation is profile-gated: the in-place writer owns single-device regular pointer extents as emitted by this package. Bucket generations are carried through a mutation: emptying a bucket advances its generation, and the alloc key, the `bucket_gens` slot, the extent pointer and its backpointer are written from one source and checked against each other before the commit is published. Foreign volumes with extra live b-trees, non-canonical `bucket_gens` or `alloc_v4` keys, inline / reflink / compressed extent keys, or unsupported inode and dirent types are refused for mutation rather than rewritten speculatively; read support is broader. How the accounting, backpointer and LRU keys were established is written up in [BCACHEFS-ACCOUNTING.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/BCACHEFS-ACCOUNTING.md). -- **ReFS** edits are offline-quiescent: the image must not be mounted, and `RefsMutationMode.NativeCow` stays fail-closed. The reader walks ReFS 3.x containers, MSB+ trees, allocators, Block Refcount rows and MLog framing; the offline editor relocates data and metadata pages, replaces and removes regular files and empty directories through CoW B+ replacement and alternate-checkpoint publication. Still open before a mounted driver could use the same core: redo-payload codecs and replay for every opcode, every Schema Table key rule, the allocation-zone policies of the three allocators, container create / delete / move, Block Refcount row creation and clone semantics, hard links, sparse and integrity-stream mutation beyond 4 KiB clusters, snapshots, ADS, security descriptors, reparse points, USN, rename, truncate, a format path, locking and a Windows `chkdsk` corpus. +- **ReFS** edits are offline-quiescent: the image must not be mounted, and `RefsMutationMode.NativeCow` stays fail-closed. The reader walks ReFS 3.x containers, MSB+ trees, allocators, Block Refcount rows and MLog framing; the offline editor relocates data and metadata pages, replaces and removes regular files and empty directories through CoW B+ replacement and alternate-checkpoint publication. Block Refcount row creation and whole-file clone semantics are implemented and tested: `RefsOfflineBlockCloner.CloneWholeFile` repoints an existing destination stream at an existing, equal-sized, cluster-aligned source stream, materialises the shared-cluster reference counts — including the first clone of a slot the sparse table does not yet track — and publishes Object Table root #0 and Block Refcount root #6 through one alternate checkpoint, reclaiming the destination's former allocation only once that checkpoint is durable. Writing to a clone afterwards reallocates rather than overwriting the shared clusters. **No public verb reaches this primitive**: it is a library entry point, not something `cwb` can be asked to do, and the profile is deliberately narrower than `FSCTL_DUPLICATE_EXTENTS_TO_FILE` — resident, sparse, integrity, zero-length, already-shared and partial-range clones are all refused. Still open before a mounted driver could use the same core: redo-payload codecs and replay for every opcode, every Schema Table key rule, the allocation-zone policies of the three allocators, container create / delete / move, partial-range and EOF-extending clones, hard links, sparse and integrity-stream mutation beyond 4 KiB clusters, snapshots, ADS, security descriptors, reparse points, USN, rename, truncate, a format path, locking and a Windows `chkdsk` corpus. - **Reiser4** remains WORM: the current writer captures the empty native tree but stores added files in a payload area the reiser4 driver does not see. It must not be promoted until the native tree writer owns those files. - **GFS2** R/W is the standalone `lock_nolock` profile with regular files in the root directory; small files are stuffed and larger files use the indirect tree. Existing-image edits rebuild the volume while preserving its size floor and lock-table value. ExHash/nested-directory writing, journal replay, cluster-lock-manager state and extended attributes remain out of scope. - **NSS** native read support is intentionally narrow and quiescent: the reader reconstructs validated DirH/LEAF namespace relationships and ordinary contiguous file extents using the OES-derived clean-room profile in [NSS-ON-DISK.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/NSS-ON-DISK.md), while retaining anchor diagnostics for unsupported images. Native allocation trees, journals, trustee metadata, ZLBH history, compressed/sparse/fragmented data profiles and writable NSS pools remain out of scope; the existing writer still emits a private Workbench container rather than a pool NetWare/OES would mount.