From c915d7e0b08779de966fc7b13065340b963f8341 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Thu, 10 Sep 2026 13:05:07 +0200 Subject: [PATCH 001/165] ReFS: add fresh block-refcount row builder --- .../FileSystem.Refs/RefsBlockRefcountCodec.cs | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs b/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs index 11a1f2f62..219f9a79d 100644 --- a/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs +++ b/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs @@ -4,9 +4,10 @@ namespace FileSystem.Refs; /// -/// Exact normal-row codec for root #6 / schema 0xe0b0. The modification stamp -/// at +0x10 and trailing dword at +0x81c are intentionally preserved verbatim; -/// their mutation semantics are not guessed. +/// Exact normal-row codec for root #6 / schema 0xe0b0. Existing-row updates +/// preserve the modification stamp at +0x10 and trailing dword at +0x81c +/// verbatim. Fresh rows require the caller to supply the checkpoint-derived +/// modification stamp explicitly. /// internal static class RefsBlockRefcountCodec { public const int EntriesPerRow = 0x400; @@ -73,12 +74,24 @@ public static bool IsUnflaggedZeroRow(ReadOnlySpan value) { } public static byte[] BuildKey(ulong startVirtualLcn) { + ValidateRangeStart(startVirtualLcn); var key = new byte[16]; BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(0, 8), startVirtualLcn); BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), EntriesPerRow); return key; } + public static byte[] BuildFreshValue(ulong startVirtualLcn, ulong modificationStamp) { + ValidateRangeStart(startVirtualLcn); + var value = new byte[NormalValueSize]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x00, 8), startVirtualLcn); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x08, 8), EntriesPerRow); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x10, 8), modificationStamp); + // TotalRefCount and the trailing dword start at zero. Per-cluster words are + // likewise zero until the caller materializes the first shared references. + return value; + } + public static void RefreshTotal(Span value) { if (value.Length < NormalValueSize) throw new InvalidDataException("ReFS Block Refcount value is shorter than 0x820 bytes."); @@ -89,6 +102,14 @@ public static void RefreshTotal(Span value) { BinaryPrimitives.WriteUInt32LittleEndian(value.Slice(0x18, 4), sum); } + private static void ValidateRangeStart(ulong startVirtualLcn) { + if ((startVirtualLcn & (EntriesPerRow - 1UL)) != 0) + throw new ArgumentOutOfRangeException( + nameof(startVirtualLcn), + startVirtualLcn, + "ReFS Block Refcount rows must start on a 0x400-cluster boundary."); + } + private static void ValidateIndex(ReadOnlySpan value, int index) { if (value.Length < NormalValueSize) throw new InvalidDataException("ReFS Block Refcount value is shorter than 0x820 bytes."); From f03381f671da1e915a471d8819787934bac09966 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Thu, 10 Sep 2026 13:05:33 +0200 Subject: [PATCH 002/165] ReFS: materialize missing block-refcount rows --- .../RefsCowBlockRefcountEditor.cs | 69 +++++++++++++++---- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs b/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs index ec54c282d..ec90cd77e 100644 --- a/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs +++ b/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs @@ -4,10 +4,14 @@ namespace FileSystem.Refs; /// -/// Native CoW editor for the populated Block Refcount tree. Existing normal -/// rows can be incremented/decremented and are removed when every count/flag -/// reaches zero. Creating a brand-new 0x820 row remains fail-closed because the -/// +0x10 modification-stamp creation rule is not yet proven. +/// Native CoW editor for root #6 / schema 0xe0b0. Existing normal rows can be +/// incremented/decremented, new normal rows are materialized from the active +/// checkpoint clock, and rows are removed when every count/flag reaches zero. +/// +/// The Block Refcount tree is sparse: an untracked live data cluster is treated +/// as having one implicit ordinary owner. Adding the first clone therefore +/// materializes count 2, both when the containing 0x400 row is absent and when +/// the row exists for unrelated shared clusters but the target slot is zero. /// internal sealed class RefsCowBlockRefcountEditor { private readonly RefsMetadataReader _metadata; @@ -51,13 +55,14 @@ public RefsCowTreeResult AdjustPhysicalReferences( return this._tree.Rewrite( this._metadata.Roots[6], virtualAddresses: true, - (rows, _) => ApplyChanges(rows, changes)); + (rows, comparer) => this.ApplyChanges(rows, comparer, changes)); } - private static bool ApplyChanges( + private bool ApplyChanges( List rows, + RefsKeyComparer comparer, IReadOnlyDictionary> changes) { - foreach (var (rowStart, deltas) in changes.OrderBy(item => item.Key)) { + foreach (var (rowStart, requestedDeltas) in changes.OrderBy(item => item.Key)) { var matching = new List(); for (var i = 0; i < rows.Count; ++i) { var candidate = rows[i]; @@ -68,15 +73,25 @@ private static bool ApplyChanges( matching.Add(i); } + if (matching.Count > 1) + throw new InvalidDataException($"ReFS Block Refcount contains duplicate row key 0x{rowStart:X}+0x400."); + if (matching.Count == 0) { - if (deltas.Values.Any(delta => delta < 0)) + if (requestedDeltas.Values.Any(delta => delta < 0)) throw new InvalidDataException( $"ReFS Block Refcount decrement targets untracked VLCN range 0x{rowStart:X}+0x400."); - throw new NotSupportedException( - $"ReFS Block Refcount row creation for VLCN range 0x{rowStart:X}+0x400 requires the unresolved +0x10 creation-stamp rule."); + + var key = RefsBlockRefcountCodec.BuildKey(rowStart); + var fresh = RefsBlockRefcountCodec.BuildFreshValue( + rowStart, + checked(this._metadata.ActiveCheckpointClock + 1)); + var deltas = requestedDeltas.ToDictionary( + item => item.Key, + item => checked(item.Value + 1)); // materialize the implicit ordinary owner + var changed = RefsBlockRefcountCodec.AdjustCounts(fresh, deltas); + rows.Insert(FindInsertion(rows, key, comparer), new RefsTreeRow(key, changed)); + continue; } - if (matching.Count != 1) - throw new InvalidDataException($"ReFS Block Refcount contains duplicate row key 0x{rowStart:X}+0x400."); var rowIndex = matching[0]; var row = rows[rowIndex]; @@ -87,12 +102,36 @@ private static bool ApplyChanges( throw new NotSupportedException( $"ReFS Block Refcount range 0x{rowStart:X}+0x400 is not a writable normal 0x820-byte row."); - var changed = RefsBlockRefcountCodec.AdjustCounts(row.Value, deltas); - if (RefsBlockRefcountCodec.IsUnflaggedZeroRow(changed)) + var deltas = new Dictionary(requestedDeltas.Count); + foreach (var (index, delta) in requestedDeltas) { + var raw = RefsBlockRefcountCodec.ReadRaw(row.Value, index); + var count = raw & RefsBlockRefcountCodec.CountMask; + var flags = raw & ~RefsBlockRefcountCodec.CountMask; + deltas[index] = delta > 0 && count == 0 && flags == 0 + ? checked(delta + 1) // zero slot is sparse-table shorthand for one ordinary owner + : delta; + } + + var updated = RefsBlockRefcountCodec.AdjustCounts(row.Value, deltas); + if (RefsBlockRefcountCodec.IsUnflaggedZeroRow(updated)) rows.RemoveAt(rowIndex); else - rows[rowIndex] = row with { Value = changed }; + rows[rowIndex] = row with { Value = updated }; } return true; } + + private static int FindInsertion( + IReadOnlyList rows, + byte[] key, + RefsKeyComparer comparer) { + var lo = 0; + var hi = rows.Count; + while (lo < hi) { + var mid = lo + ((hi - lo) >> 1); + if (comparer.Compare(rows[mid].Key, key) < 0) lo = mid + 1; + else hi = mid; + } + return lo; + } } From ff552c930eaa9eac49844776aea31fca0069adf6 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Thu, 10 Sep 2026 13:05:41 +0200 Subject: [PATCH 003/165] ReFS: remove duplicate refcount mutation policy --- .../RefsBlockRefcountPolicy.cs | 188 ------------------ 1 file changed, 188 deletions(-) delete mode 100644 FileSystems/FileSystem.Refs/RefsBlockRefcountPolicy.cs diff --git a/FileSystems/FileSystem.Refs/RefsBlockRefcountPolicy.cs b/FileSystems/FileSystem.Refs/RefsBlockRefcountPolicy.cs deleted file mode 100644 index 56f55f8da..000000000 --- a/FileSystems/FileSystem.Refs/RefsBlockRefcountPolicy.cs +++ /dev/null @@ -1,188 +0,0 @@ -#pragma warning disable CS1591 -using System.Buffers.Binary; - -namespace FileSystem.Refs; - -/// -/// Functional mutation policy for root #6. It can materialize a previously -/// absent 0x400-cluster row when a singly-owned extent becomes shared, increment -/// existing entries, preserve the two high management flags, and rewrite the -/// TotalRefCount invariant before publishing a CoW replacement root. -/// -internal sealed class RefsBlockRefcountPolicy { - private const ulong RangeClusters = 0x400; - private const int ValueSize = 0x820; - private const int CountsOffset = 0x1C; - private const ushort CountMask = 0x3FFF; - private const ushort FlagsMask = 0xC000; - - private readonly RefsMetadataReader _metadata; - private readonly RefsCowBTree _btree; - - public RefsBlockRefcountPolicy(RefsMetadataReader metadata, RefsCowBTree btree) { - this._metadata = metadata; - this._btree = btree; - } - - /// - /// Adds references to already-live clusters. An untracked cluster is assumed - /// to have its ordinary single live owner; adding one clone therefore creates - /// a refcount of 2 rather than 1. - /// - public RefsCowTreeResult IncrementPhysicalReferences( - IEnumerable physicalClusters, - ushort additionalReferences = 1, - ushort managementFlags = 0) { - if (additionalReferences == 0) throw new ArgumentOutOfRangeException(nameof(additionalReferences)); - if ((managementFlags & ~FlagsMask) != 0) throw new ArgumentOutOfRangeException(nameof(managementFlags)); - var requested = BuildRequest(physicalClusters); - return this.Rewrite(requested, (current, existed) => { - var flags = (ushort)((current & FlagsMask) | managementFlags); - var count = current & CountMask; - var next = existed - ? checked(count + additionalReferences) - : checked(1 + additionalReferences); - if (next > CountMask) throw new OverflowException("ReFS block refcount exceeds its 14-bit on-disk field."); - return (ushort)(flags | next); - }); - } - - public RefsCowTreeResult ClonePhysicalReferences(IEnumerable physicalClusters) - => this.IncrementPhysicalReferences(physicalClusters, additionalReferences: 1); - - public RefsCowTreeResult DecrementPhysicalReferences( - IEnumerable physicalClusters, - ushort references = 1) { - if (references == 0) throw new ArgumentOutOfRangeException(nameof(references)); - var requested = BuildRequest(physicalClusters); - return this.Rewrite(requested, (current, existed) => { - if (!existed) throw new InvalidOperationException("ReFS cannot decrement an untracked shared cluster."); - var flags = (ushort)(current & FlagsMask); - var count = current & CountMask; - if (count < references) - throw new InvalidDataException("ReFS block-refcount underflow would occur."); - return (ushort)(flags | (count - references)); - }); - } - - private RefsCowTreeResult Rewrite( - IReadOnlyDictionary> request, - Func mutate) { - if (this._metadata.Roots.Count <= 6 || this._metadata.Roots[6].Lcns.Count == 0) - throw new NotSupportedException("ReFS Block Refcount root #6 is unavailable."); - - return this._btree.Rewrite(this._metadata.Roots[6], virtualAddresses: true, (rows, comparer) => { - var byStart = new Dictionary(); - for (var i = 0; i < rows.Count; ++i) { - if (!TryParseKey(rows[i].Key, out var start)) continue; - byStart[start] = i; - } - - foreach (var (rangeStart, entries) in request.OrderBy(p => p.Key)) { - RefsTreeRow row; - var rowExists = byStart.TryGetValue(rangeStart, out var rowIndex); - if (rowExists) { - row = rows[rowIndex]; - if (row.Value.Length != ValueSize) - throw new NotSupportedException( - $"ReFS Block Refcount row 0x{rangeStart:X} uses unsupported {row.Value.Length}-byte value form."); - ValidateValue(row.Value, rangeStart); - } else { - var key = BuildKey(rangeStart); - var value = BuildEmptyValue(rangeStart, this._metadata.ActiveCheckpointClock); - row = new RefsTreeRow(key, value, 0); - rowIndex = ~FindInsertion(rows, key, comparer); - if (rowIndex < 0) throw new IOException("ReFS refcount row key unexpectedly already exists."); - rows.Insert(rowIndex, row); - byStart[rangeStart] = rowIndex; - // insertion shifts following indices; rebuild the map once for safety. - byStart.Clear(); - for (var i = 0; i < rows.Count; ++i) - if (TryParseKey(rows[i].Key, out var s)) byStart[s] = i; - rowIndex = byStart[rangeStart]; - } - - var updated = row.Value.ToArray(); - foreach (var (slot, _) in entries) { - var offset = CountsOffset + slot * 2; - var current = BinaryPrimitives.ReadUInt16LittleEndian(updated.AsSpan(offset, 2)); - var existed = rowExists && (current & CountMask) != 0; - var next = mutate(current, existed); - BinaryPrimitives.WriteUInt16LittleEndian(updated.AsSpan(offset, 2), next); - } - BinaryPrimitives.WriteUInt64LittleEndian(updated.AsSpan(0x10, 8), this._metadata.ActiveCheckpointClock + 1); - RefreshTotal(updated); - rows[rowIndex] = row with { Value = updated }; - } - return true; - }); - } - - private IReadOnlyDictionary> BuildRequest(IEnumerable physicalClusters) { - ArgumentNullException.ThrowIfNull(physicalClusters); - var result = new Dictionary>(); - foreach (var physical in physicalClusters.Distinct()) { - if (!this._metadata.TryPhysicalToVirtualLcn(physical, out var vlcn)) - throw new InvalidDataException($"ReFS shared data PLCN 0x{physical:X} has no VLCN mapping."); - var rangeStart = vlcn & ~(RangeClusters - 1); - var index = checked((int)(vlcn - rangeStart)); - if (!result.TryGetValue(rangeStart, out var entries)) result[rangeStart] = entries = []; - entries[index] = physical; - } - if (result.Count == 0) throw new ArgumentException("No ReFS clusters were supplied.", nameof(physicalClusters)); - return result; - } - - private static byte[] BuildKey(ulong start) { - var key = new byte[16]; - BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(0, 8), start); - BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), RangeClusters); - return key; - } - - private static byte[] BuildEmptyValue(ulong start, ulong stamp) { - var value = new byte[ValueSize]; - BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x00, 8), start); - BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x08, 8), RangeClusters); - BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x10, 8), stamp); - return value; - } - - private static bool TryParseKey(byte[] key, out ulong start) { - start = 0; - if (key.Length < 16 || BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(8, 8)) != RangeClusters) return false; - start = BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(0, 8)); - return true; - } - - private static void ValidateValue(byte[] value, ulong expectedStart) { - if (BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(0x00, 8)) != expectedStart - || BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(0x08, 8)) != RangeClusters) - throw new InvalidDataException("ReFS Block Refcount value does not echo its key range."); - uint total = 0; - for (ulong i = 0; i < RangeClusters; ++i) - total = checked(total + (uint)(BinaryPrimitives.ReadUInt16LittleEndian(value.AsSpan(CountsOffset + checked((int)i) * 2, 2)) & CountMask)); - if (BinaryPrimitives.ReadUInt32LittleEndian(value.AsSpan(0x18, 4)) != total) - throw new InvalidDataException("ReFS Block Refcount TotalRefCount does not match its per-cluster array."); - } - - private static void RefreshTotal(Span value) { - uint total = 0; - for (ulong i = 0; i < RangeClusters; ++i) - total = checked(total + (uint)(BinaryPrimitives.ReadUInt16LittleEndian(value.Slice(CountsOffset + checked((int)i) * 2, 2)) & CountMask)); - BinaryPrimitives.WriteUInt32LittleEndian(value.Slice(0x18, 4), total); - } - - private static int FindInsertion(IReadOnlyList rows, byte[] key, RefsKeyComparer comparer) { - var lo = 0; - var hi = rows.Count - 1; - while (lo <= hi) { - var mid = lo + ((hi - lo) >> 1); - var cmp = comparer.Compare(rows[mid].Key, key); - if (cmp == 0) return mid; - if (cmp < 0) lo = mid + 1; - else hi = mid - 1; - } - return ~lo; - } -} From 535754789a3a51a2694c1e0c9f68655c4c1c3c6f Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Thu, 10 Sep 2026 13:08:42 +0200 Subject: [PATCH 004/165] ReFS: add offline whole-file block clone --- .../FileSystem.Refs/RefsOfflineBlockCloner.cs | 256 ++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs diff --git a/FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs b/FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs new file mode 100644 index 000000000..7a0d5eb55 --- /dev/null +++ b/FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs @@ -0,0 +1,256 @@ +#pragma warning disable CS1591 + +namespace FileSystem.Refs; + +internal sealed record RefsOfflineCloneResult( + long LogicalSize, + int SharedClusterCount, + ulong CheckpointPhysicalLcn, + ulong CheckpointClock); + +/// +/// Performs the bounded offline ReFS block-clone operation for two existing, +/// equal-sized ordinary extent-backed files. The destination's stream mapping is +/// replaced by the source mapping through immutable object/Object-Table CoW, +/// while root #6 gains the corresponding shared-cluster reference counts in the +/// same alternate-checkpoint publication. +/// +/// This is deliberately narrower than FSCTL_DUPLICATE_EXTENTS_TO_FILE: it clones +/// a whole cluster-aligned file only, does not create names or extend EOF, and +/// refuses resident, sparse, integrity, already-shared and otherwise undecoded +/// stream layouts. Native mounted-driver/MLog semantics remain fail-closed. +/// +internal static class RefsOfflineBlockCloner { + public static RefsOfflineCloneResult CloneWholeFile( + Stream image, + string sourcePath, + string destinationPath) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("ReFS offline block clone requires a readable, writable, seekable image.", nameof(image)); + + sourcePath = NormalizePath(sourcePath, nameof(sourcePath)); + destinationPath = NormalizePath(destinationPath, nameof(destinationPath)); + if (sourcePath.Equals(destinationPath, StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException("ReFS block clone source and destination must be different files.", nameof(destinationPath)); + + var metadata = RefsMetadataReader.Open(image); + var files = new RefsNamespaceReader(metadata).ReadAll(); + var source = FindRegularFile(files, sourcePath, nameof(sourcePath)); + var destination = FindRegularFile(files, destinationPath, nameof(destinationPath)); + ValidateWholeFileProfile(metadata, source, destination); + + var sourcePhysical = ExpandPhysicalClusters(metadata, source); + var destinationPhysical = ExpandPhysicalClusters(metadata, destination); + if (sourcePhysical.Count != destinationPhysical.Count) + throw new InvalidDataException("ReFS whole-file clone source and destination have different allocation coverage."); + if (sourcePhysical.Count == 0) + throw new NotSupportedException("ReFS zero-length files do not need block-clone metadata."); + if (sourcePhysical.Distinct().Count() != sourcePhysical.Count) + throw new NotSupportedException("ReFS source file already aliases a physical cluster inside its own extent map."); + if (destinationPhysical.Distinct().Count() != destinationPhysical.Count) + throw new NotSupportedException("ReFS destination file already aliases a physical cluster inside its own extent map."); + if (sourcePhysical.Intersect(destinationPhysical).Any()) + throw new NotSupportedException("ReFS whole-file clone refuses source/destination mappings that already overlap."); + + var refcounts = new RefsBlockRefcount(metadata); + EnsureUnshared(refcounts, sourcePhysical, "source"); + EnsureUnshared(refcounts, destinationPhysical, "destination"); + + var sourceOffsets = sourcePhysical + .Select(lcn => checked((long)lcn * metadata.ClusterSize)) + .ToArray(); + var clonedExtents = RefsStreamLayoutEditor.BuildExtents(metadata, sourceOffsets); + + var store = new RefsCowPageStore(image, metadata); + var tree = new RefsCowBTree(image, metadata, store); + var objectEditor = new RefsCowObjectEditor(metadata, tree); + var objectMutation = objectEditor.UpdateStorageValue( + destinationPath, + location => RefsStreamLayoutEditor.BuildUpdatedValue( + destination, + location.StorageRow, + clonedExtents, + metadata.ClusterSize)); + + var refcountTree = new RefsCowBlockRefcountEditor(metadata, tree) + .IncrementPhysicalReferences(sourcePhysical); + + var checkpoint = PublishOfflineCheckpoint( + image, + metadata, + store, + new Dictionary { + [0] = objectMutation.ObjectTableTree.RootReference, + [6] = refcountTree.RootReference, + }); + + // The clone is already durable at this point. Releasing the destination's + // former ordinary allocation is post-commit reclamation; inability to prove + // allocator ownership leaves a safe leak rather than risking live data. + TryReleaseFormerDestination(image, destinationPhysical); + VerifyClone(image, sourcePath, destinationPath, sourcePhysical); + + return new RefsOfflineCloneResult( + source.Size, + sourcePhysical.Count, + checkpoint.ActiveCheckpointLcn, + checkpoint.ActiveCheckpointClock); + } + + private static RefsMetadataReader PublishOfflineCheckpoint( + Stream image, + RefsMetadataReader metadata, + RefsCowPageStore store, + IReadOnlyDictionary replacements) { + if (!replacements.ContainsKey(0) || !replacements.ContainsKey(6)) + throw new ArgumentException("ReFS block clone must publish Object Table root #0 and Block Refcount root #6 together.", nameof(replacements)); + + var roots = replacements.ToDictionary(item => item.Key, item => item.Value.ToArray()); + var allocatorChanged = false; + foreach (var tier in new[] { + RefsAllocatorTier.Medium, + RefsAllocatorTier.Container, + RefsAllocatorTier.Small, + }) { + if (store.GetReservedClusters(tier).Count == 0) continue; + var publication = new RefsCowAllocatorPublisher(image, metadata, store).Publish(tier); + roots[publication.RootIndex] = publication.Tree.RootReference; + allocatorChanged = true; + } + + var committer = new RefsCheckpointCommitter(image); + var prepared = committer.PrepareNext(); + committer.SetRootReferences(prepared, roots); + committer.Commit(prepared, allocatorChanged: allocatorChanged); + image.Flush(); + + var verify = RefsMetadataReader.Open(image); + foreach (var rootIndex in new[] { 0, 6 }) { + var expected = RefsPageReference.Parse(roots[rootIndex]); + var actual = verify.Roots[rootIndex]; + if (!expected.Lcns.SequenceEqual(actual.Lcns)) + throw new IOException($"ReFS clone checkpoint published but root #{rootIndex} does not match the replacement tree."); + } + return verify; + } + + private static void VerifyClone( + Stream image, + string sourcePath, + string destinationPath, + IReadOnlyList expectedPhysical) { + var metadata = RefsMetadataReader.Open(image); + var files = new RefsNamespaceReader(metadata).ReadAll(); + var source = FindRegularFile(files, sourcePath, nameof(sourcePath)); + var destination = FindRegularFile(files, destinationPath, nameof(destinationPath)); + var sourcePhysical = ExpandPhysicalClusters(metadata, source); + var destinationPhysical = ExpandPhysicalClusters(metadata, destination); + if (!sourcePhysical.SequenceEqual(expectedPhysical) || !destinationPhysical.SequenceEqual(expectedPhysical)) + throw new IOException("ReFS block clone checkpoint is live but source/destination extent mappings do not match."); + + var refcounts = new RefsBlockRefcount(metadata); + foreach (var physical in expectedPhysical) { + if (!refcounts.TryGetPhysical(physical, out var entry) || entry.ReferenceCount != 2) + throw new IOException($"ReFS block clone PLCN 0x{physical:X} does not have the expected reference count 2."); + } + } + + private static void TryReleaseFormerDestination(Stream image, IReadOnlyList physicalClusters) { + try { + var metadata = RefsMetadataReader.Open(image); + var graph = new RefsMetadataGraph(image, metadata); + var remaining = new RefsBlockRefcount(metadata, graph) + .DetachPhysicalReferences(physicalClusters) + .ToHashSet(); + if (remaining.Count == 0) return; + + foreach (var tier in new[] { + RefsAllocatorTier.Medium, + RefsAllocatorTier.Container, + RefsAllocatorTier.Small, + }) { + var writer = new RefsAllocatorWriter(metadata, graph, tier); + var covered = remaining.Where(writer.CoversPhysical).ToArray(); + if (covered.Length == 0) continue; + writer.SetAllocated(covered, allocated: false); + foreach (var lcn in covered) remaining.Remove(lcn); + image.Flush(); + } + // Unknown ownership remains allocated deliberately. + } catch (Exception e) when (e is InvalidDataException or NotSupportedException or IOException or InvalidOperationException) { + // The metadata clone was committed first. Reclamation failure is a safe + // allocation leak and must not turn a durable clone into a false rollback. + } + } + + private static void ValidateWholeFileProfile( + RefsMetadataReader metadata, + RefsFileRecord source, + RefsFileRecord destination) { + if (source.IsResident || destination.IsResident) + throw new NotSupportedException("ReFS whole-file block clone currently requires two extent-backed files."); + if (source.Size != destination.Size) + throw new NotSupportedException("ReFS whole-file block clone requires equal source/destination logical sizes; EOF extension is not implemented."); + if (source.Size <= 0 || source.Size % metadata.ClusterSize != 0) + throw new NotSupportedException("ReFS whole-file block clone currently requires a non-empty cluster-aligned logical size."); + if (source.AllocatedSize != source.Size || destination.AllocatedSize != destination.Size) + throw new NotSupportedException("ReFS whole-file block clone requires fully allocated, non-sparse stream coverage."); + if (source.Extents.Count == 0 || destination.Extents.Count == 0) + throw new InvalidDataException("ReFS extent-backed clone candidate has no decoded data extents."); + if (source.Extents.Any(IsUnsupportedExtent) || destination.Extents.Any(IsUnsupportedExtent)) + throw new NotSupportedException("ReFS whole-file block clone refuses sparse, integrity, snapshot/shared or otherwise special extents."); + } + + private static bool IsUnsupportedExtent(RefsDataExtent extent) + => extent.IsSparse || extent.Flags == 0x1C00D0 || (extent.Flags & 0x04) != 0; + + private static IReadOnlyList ExpandPhysicalClusters( + RefsMetadataReader metadata, + RefsFileRecord file) { + var result = new List(); + uint expectedVcn = 0; + foreach (var extent in file.Extents.OrderBy(item => item.FileVcn)) { + if (extent.FileVcn != expectedVcn) + throw new InvalidDataException($"ReFS file '{file.Path}' extent map is not a contiguous VCN cover."); + for (uint i = 0; i < extent.ClusterCount; ++i) + result.Add(metadata.TranslateVirtualLcn(checked(extent.VirtualLcn + i))); + expectedVcn = checked(expectedVcn + extent.ClusterCount); + } + + var expectedClusters = checked((int)(file.Size / metadata.ClusterSize)); + if (result.Count != expectedClusters) + throw new InvalidDataException( + $"ReFS file '{file.Path}' extent map covers {result.Count} cluster(s), expected {expectedClusters}."); + return result; + } + + private static void EnsureUnshared( + RefsBlockRefcount refcounts, + IEnumerable physicalClusters, + string role) { + foreach (var physical in physicalClusters) + if (refcounts.TryGetPhysical(physical, out var entry) && entry.IsShared) + throw new NotSupportedException( + $"ReFS whole-file clone {role} PLCN 0x{physical:X} is already shared/dedup-managed; repeated clone integration is a separate profile."); + } + + private static RefsFileRecord FindRegularFile( + IEnumerable files, + string path, + string parameterName) { + var file = files.FirstOrDefault(item => string.Equals(item.Path, path, StringComparison.OrdinalIgnoreCase)) + ?? throw new FileNotFoundException($"ReFS file '{path}' was not found.", path); + if (file.IsDirectory) + throw new ArgumentException($"ReFS path '{path}' names a directory.", parameterName); + return file; + } + + private static string NormalizePath(string path, string parameterName) { + ArgumentException.ThrowIfNullOrWhiteSpace(path, parameterName); + var parts = path.Replace('\\', '/').Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 0 || parts.Any(part => part is "." or ".." || part.IndexOf('\0') >= 0)) + throw new ArgumentException("ReFS clone path must be a canonical relative filesystem path.", parameterName); + return string.Join('/', parts); + } +} From 816482b2f2830e9aa6758f6b838f189e610deed5 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Thu, 10 Sep 2026 13:09:55 +0200 Subject: [PATCH 005/165] ReFS: expose bounded offline block clone API --- FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs b/FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs index 7a0d5eb55..622a62cde 100644 --- a/FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs +++ b/FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs @@ -20,8 +20,15 @@ internal sealed record RefsOfflineCloneResult( /// refuses resident, sparse, integrity, already-shared and otherwise undecoded /// stream layouts. Native mounted-driver/MLog semantics remain fail-closed. /// -internal static class RefsOfflineBlockCloner { - public static RefsOfflineCloneResult CloneWholeFile( +public static class RefsOfflineBlockCloner { + /// + /// Replaces the contents of an existing destination file with a metadata-only + /// clone of an existing source file on an unmounted ReFS image. + /// + public static void CloneWholeFile(Stream image, string sourcePath, string destinationPath) + => CloneWholeFileCore(image, sourcePath, destinationPath); + + internal static RefsOfflineCloneResult CloneWholeFileCore( Stream image, string sourcePath, string destinationPath) { From 74c6da67c83504f3b3f81e35e031d4102e148d1b Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Thu, 10 Sep 2026 13:10:23 +0200 Subject: [PATCH 006/165] ReFS: test fresh block-refcount rows --- .../Refs/RefsBlockRefcountCodecTests.cs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs b/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs index 821561f57..f59f705ca 100644 --- a/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs +++ b/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs @@ -80,6 +80,45 @@ public void BuildKey_UsesAlignedRangeAndFixedCount() { }); } + [Test, Category("HappyPath")] + public void BuildFreshValue_InitializesRangeStampAndZeroTotal() { + var row = RefsBlockRefcountCodec.BuildFreshValue(0x123400UL, modificationStamp: 0xE5); + + Assert.Multiple(() => { + Assert.That(row, Has.Length.EqualTo(0x820)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(row.AsSpan(0x00, 8)), Is.EqualTo(0x123400UL)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(row.AsSpan(0x08, 8)), Is.EqualTo(0x400UL)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(row.AsSpan(0x10, 8)), Is.EqualTo(0xE5UL)); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(row.AsSpan(0x18, 4)), Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(row.AsSpan(0x81C, 4)), Is.Zero); + Assert.That(RefsBlockRefcountCodec.HasValidTotal(row), Is.True); + Assert.That(RefsBlockRefcountCodec.IsUnflaggedZeroRow(row), Is.True); + }); + } + + [Test, Category("HappyPath")] + public void FreshValue_FirstCloneCanMaterializeImplicitOwnerAsCountTwo() { + var row = RefsBlockRefcountCodec.BuildFreshValue(0x4000UL, modificationStamp: 0x99); + + var changed = RefsBlockRefcountCodec.AdjustCounts( + row, + new Dictionary { [7] = +2 }); + + Assert.Multiple(() => { + Assert.That(RefsBlockRefcountCodec.ReadCount(changed, 7), Is.EqualTo(2)); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(changed.AsSpan(0x18, 4)), Is.EqualTo(2)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(changed.AsSpan(0x10, 8)), Is.EqualTo(0x99UL)); + }); + } + + [Test, Category("ErrorHandling")] + public void FreshRowBuilders_RejectUnalignedRangeStart() { + Assert.Multiple(() => { + Assert.Throws(() => RefsBlockRefcountCodec.BuildKey(0x123401UL)); + Assert.Throws(() => RefsBlockRefcountCodec.BuildFreshValue(0x123401UL, 1)); + }); + } + private static byte[] BuildRow(ulong start) { var row = new byte[RefsBlockRefcountCodec.NormalValueSize]; BinaryPrimitives.WriteUInt64LittleEndian(row.AsSpan(0, 8), start); From d99dcbaa74060fcdc7b0ead75f71f89b60182591 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Thu, 10 Sep 2026 13:12:46 +0200 Subject: [PATCH 007/165] ReFS: centralize first-clone refcount semantics --- .../FileSystem.Refs/RefsBlockRefcountCodec.cs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs b/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs index 219f9a79d..5dde4eef6 100644 --- a/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs +++ b/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs @@ -66,6 +66,32 @@ public static byte[] AdjustCounts( return result; } + /// + /// Adds clone references to a sparse Block Refcount row. A completely zero + /// slot is not an unowned cluster: absence from the sparse table represents + /// the ordinary single allocation owner. Its first clone therefore adds that + /// implicit owner before the requested new references. Flagged zero-count + /// slots belong to a separate management class and are not given that baseline. + /// + public static byte[] AddCloneReferences( + ReadOnlySpan original, + IReadOnlyDictionary additionalReferences) { + ArgumentNullException.ThrowIfNull(additionalReferences); + if (!TryGetRange(original, out _, out _) || !HasValidTotal(original)) + throw new InvalidDataException("ReFS Block Refcount row is not a valid normal 0x820-byte row."); + + var deltas = new Dictionary(additionalReferences.Count); + foreach (var (index, additional) in additionalReferences) { + if (additional <= 0) + throw new ArgumentOutOfRangeException(nameof(additionalReferences), "ReFS clone references must be positive."); + var raw = ReadRaw(original, index); + var count = raw & CountMask; + var flags = raw & ~CountMask; + deltas[index] = checked(additional + (count == 0 && flags == 0 ? 1 : 0)); + } + return AdjustCounts(original, deltas); + } + public static bool IsUnflaggedZeroRow(ReadOnlySpan value) { if (!TryGetRange(value, out _, out _) || !HasValidTotal(value)) return false; for (var i = 0; i < EntriesPerRow; ++i) From f1b7ab2f860f61fb2a4c0315dd016b6a9d165cf8 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Thu, 10 Sep 2026 13:13:17 +0200 Subject: [PATCH 008/165] ReFS: reuse clone-aware refcount codec --- .../RefsCowBlockRefcountEditor.cs | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs b/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs index ec90cd77e..534be899a 100644 --- a/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs +++ b/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs @@ -85,10 +85,7 @@ private bool ApplyChanges( var fresh = RefsBlockRefcountCodec.BuildFreshValue( rowStart, checked(this._metadata.ActiveCheckpointClock + 1)); - var deltas = requestedDeltas.ToDictionary( - item => item.Key, - item => checked(item.Value + 1)); // materialize the implicit ordinary owner - var changed = RefsBlockRefcountCodec.AdjustCounts(fresh, deltas); + var changed = RefsBlockRefcountCodec.AddCloneReferences(fresh, requestedDeltas); rows.Insert(FindInsertion(rows, key, comparer), new RefsTreeRow(key, changed)); continue; } @@ -102,17 +99,9 @@ private bool ApplyChanges( throw new NotSupportedException( $"ReFS Block Refcount range 0x{rowStart:X}+0x400 is not a writable normal 0x820-byte row."); - var deltas = new Dictionary(requestedDeltas.Count); - foreach (var (index, delta) in requestedDeltas) { - var raw = RefsBlockRefcountCodec.ReadRaw(row.Value, index); - var count = raw & RefsBlockRefcountCodec.CountMask; - var flags = raw & ~RefsBlockRefcountCodec.CountMask; - deltas[index] = delta > 0 && count == 0 && flags == 0 - ? checked(delta + 1) // zero slot is sparse-table shorthand for one ordinary owner - : delta; - } - - var updated = RefsBlockRefcountCodec.AdjustCounts(row.Value, deltas); + var updated = requestedDeltas.Values.All(delta => delta > 0) + ? RefsBlockRefcountCodec.AddCloneReferences(row.Value, requestedDeltas) + : RefsBlockRefcountCodec.AdjustCounts(row.Value, requestedDeltas); if (RefsBlockRefcountCodec.IsUnflaggedZeroRow(updated)) rows.RemoveAt(rowIndex); else From 895f8e3936f83a793eec44016a4507460212f45a Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Thu, 10 Sep 2026 13:14:01 +0200 Subject: [PATCH 009/165] ReFS: cover sparse-slot clone semantics --- .../Refs/RefsBlockRefcountCodecTests.cs | 63 ++++++++++++++++++- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs b/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs index f59f705ca..928d72dca 100644 --- a/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs +++ b/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs @@ -97,12 +97,12 @@ public void BuildFreshValue_InitializesRangeStampAndZeroTotal() { } [Test, Category("HappyPath")] - public void FreshValue_FirstCloneCanMaterializeImplicitOwnerAsCountTwo() { + public void AddCloneReferences_FreshRowMaterializesImplicitOwnerAsCountTwo() { var row = RefsBlockRefcountCodec.BuildFreshValue(0x4000UL, modificationStamp: 0x99); - var changed = RefsBlockRefcountCodec.AdjustCounts( + var changed = RefsBlockRefcountCodec.AddCloneReferences( row, - new Dictionary { [7] = +2 }); + new Dictionary { [7] = 1 }); Assert.Multiple(() => { Assert.That(RefsBlockRefcountCodec.ReadCount(changed, 7), Is.EqualTo(2)); @@ -111,6 +111,63 @@ public void FreshValue_FirstCloneCanMaterializeImplicitOwnerAsCountTwo() { }); } + [Test, Category("HappyPath")] + public void AddCloneReferences_ExistingRowZeroSlotStillMaterializesCountTwo() { + var row = BuildRow(0x8000); + WriteRaw(row, 3, 2); // another shared cluster makes the sparse row already exist + RefsBlockRefcountCodec.RefreshTotal(row); + + var changed = RefsBlockRefcountCodec.AddCloneReferences( + row, + new Dictionary { [9] = 1 }); + + Assert.Multiple(() => { + Assert.That(RefsBlockRefcountCodec.ReadCount(changed, 3), Is.EqualTo(2)); + Assert.That(RefsBlockRefcountCodec.ReadCount(changed, 9), Is.EqualTo(2)); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(changed.AsSpan(0x18, 4)), Is.EqualTo(4)); + }); + } + + [Test, Category("HappyPath")] + public void AddCloneReferences_TrackedSlotIncrementsWithoutImplicitBaseline() { + var row = BuildRow(0xC000); + WriteRaw(row, 12, 2); + RefsBlockRefcountCodec.RefreshTotal(row); + + var changed = RefsBlockRefcountCodec.AddCloneReferences( + row, + new Dictionary { [12] = 1 }); + + Assert.That(RefsBlockRefcountCodec.ReadCount(changed, 12), Is.EqualTo(3)); + } + + [Test, Category("HappyPath")] + public void AddCloneReferences_FlaggedZeroSlotDoesNotAssumeOrdinaryOwner() { + var row = BuildRow(0x10000); + WriteRaw(row, 5, RefsBlockRefcountCodec.DedupManagedMask); + RefsBlockRefcountCodec.RefreshTotal(row); + + var changed = RefsBlockRefcountCodec.AddCloneReferences( + row, + new Dictionary { [5] = 1 }); + + Assert.That(RefsBlockRefcountCodec.ReadRaw(changed, 5), Is.EqualTo(0x8001)); + } + + [Test, Category("ErrorHandling")] + public void AddCloneReferences_RejectsNonPositiveReferenceCount() { + var row = BuildRow(0x14000); + + Assert.Multiple(() => { + Assert.Throws(() => RefsBlockRefcountCodec.AddCloneReferences( + row, + new Dictionary { [1] = 0 })); + Assert.Throws(() => RefsBlockRefcountCodec.AddCloneReferences( + row, + new Dictionary { [1] = -1 })); + }); + } + [Test, Category("ErrorHandling")] public void FreshRowBuilders_RejectUnalignedRangeStart() { Assert.Multiple(() => { From 3ff3ed2131f856733867efdc390d1485260e7d01 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Thu, 10 Sep 2026 13:16:14 +0200 Subject: [PATCH 010/165] ReFS: add real-image block-clone interop gate --- .../Refs/RefsBlockCloneExternalTests.cs | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 Compression.Tests/Refs/RefsBlockCloneExternalTests.cs diff --git a/Compression.Tests/Refs/RefsBlockCloneExternalTests.cs b/Compression.Tests/Refs/RefsBlockCloneExternalTests.cs new file mode 100644 index 000000000..cd74a9d17 --- /dev/null +++ b/Compression.Tests/Refs/RefsBlockCloneExternalTests.cs @@ -0,0 +1,103 @@ +#pragma warning disable CS1591 +using System.ComponentModel; +using System.Diagnostics; +using FileSystem.Refs; + +namespace Compression.Tests.Refs; + +/// +/// Optional real-image acceptance gate for the offline ReFS whole-file clone. +/// It intentionally does not download or vendor a ReFS implementation: point +/// CWB_REFS_CLONE_IMAGE at a disposable/reference raw ReFS image and +/// CWB_REFS_FOREFST at an existing forefst.py checkout. The image must contain +/// two distinct, equal-sized, non-empty, cluster-aligned ordinary extent-backed +/// files named by CWB_REFS_CLONE_SOURCE and CWB_REFS_CLONE_DESTINATION. +/// +/// The GPL-3.0-or-later forefst tool is used only as an external behavioral +/// oracle; no source or runtime dependency is incorporated into the product. +/// +[TestFixture] +[Category("ExternalFsInterop")] +public sealed class RefsBlockCloneExternalTests { + private string _tmpDir = null!; + + [SetUp] + public void SetUp() { + this._tmpDir = Path.Combine(Path.GetTempPath(), $"cwb_refs_clone_{Guid.NewGuid():N}"); + Directory.CreateDirectory(this._tmpDir); + } + + [TearDown] + public void TearDown() { + try { Directory.Delete(this._tmpDir, recursive: true); } catch { /* best effort */ } + } + + [Test, CancelAfter(180_000), Category("HappyPath")] + public void RealImage_WholeFileClone_IsReadableByForefst() { + var fixture = Environment.GetEnvironmentVariable("CWB_REFS_CLONE_IMAGE"); + var forefst = Environment.GetEnvironmentVariable("CWB_REFS_FOREFST"); + var source = Environment.GetEnvironmentVariable("CWB_REFS_CLONE_SOURCE") ?? "clone-source.bin"; + var destination = Environment.GetEnvironmentVariable("CWB_REFS_CLONE_DESTINATION") ?? "clone-destination.bin"; + + if (string.IsNullOrWhiteSpace(fixture) || !File.Exists(fixture)) + Assert.Ignore("Set CWB_REFS_CLONE_IMAGE to a real raw ReFS image containing the clone source/destination files."); + if (string.IsNullOrWhiteSpace(forefst) || !File.Exists(forefst)) + Assert.Ignore("Set CWB_REFS_FOREFST to forefst.py for the external ReFS reader oracle."); + + var image = Path.Combine(this._tmpDir, "clone.refs"); + File.Copy(fixture!, image); + var beforeSource = ExtractWithForefst(forefst!, image, source, Path.Combine(this._tmpDir, "source-before.bin")); + + using (var stream = File.Open(image, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) + RefsOfflineBlockCloner.CloneWholeFile(stream, source, destination); + + var afterSource = ExtractWithForefst(forefst!, image, source, Path.Combine(this._tmpDir, "source-after.bin")); + var afterDestination = ExtractWithForefst(forefst!, image, destination, Path.Combine(this._tmpDir, "destination-after.bin")); + + Assert.Multiple(() => { + Assert.That(afterSource, Is.EqualTo(beforeSource), "External reader sees changed source bytes after clone."); + Assert.That(afterDestination, Is.EqualTo(beforeSource), "External reader does not see destination as a byte-identical block clone."); + }); + } + + private static byte[] ExtractWithForefst( + string forefst, + string image, + string path, + string outputPath) { + var python = Environment.GetEnvironmentVariable("CWB_PYTHON") + ?? (OperatingSystem.IsWindows() ? "python" : "python3"); + var start = new ProcessStartInfo { + FileName = python, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + start.ArgumentList.Add(forefst); + start.ArgumentList.Add(image); + start.ArgumentList.Add("extract"); + start.ArgumentList.Add(path); + + Process process; + try { + process = Process.Start(start) + ?? throw new InvalidOperationException("Could not start the forefst process."); + } catch (Win32Exception) { + Assert.Ignore($"Python executable '{python}' is unavailable for the forefst oracle."); + return []; + } + + using (process) { + var stderr = process.StandardError.ReadToEndAsync(); + using (var output = File.Create(outputPath)) + process.StandardOutput.BaseStream.CopyTo(output); + process.WaitForExit(); + var error = stderr.GetAwaiter().GetResult(); + Assert.That(process.ExitCode, Is.Zero, + $"forefst could not extract '{path}' from the ReFS image:\n{error}"); + } + + return File.ReadAllBytes(outputPath); + } +} From 9e8eea5c6095ffebe788354f487323ff7bdb3ea1 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:56:10 +0200 Subject: [PATCH 011/165] Preserve Intel HEX segmented start addresses --- FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs b/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs index bc2710aa7..23b02dcd3 100644 --- a/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs +++ b/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs @@ -17,6 +17,14 @@ public sealed record FirmwareImage( string SourceFormat ) { + /// + /// Gets the original segmented x86 start address when an Intel HEX type-03 + /// record supplied one. Keeping CS:IP separately matters because many CS:IP + /// pairs map to the same linear address and therefore cannot be reconstructed + /// from alone. + /// + public (ushort CodeSegment, ushort InstructionPointer)? StartSegmentAddress { get; init; } + /// /// Flattens all segments into a single contiguous binary spanning from the /// lowest address to the end of the highest segment. Gaps are filled with From 5ca1a4cdb17fe1c04f2ce56a3a6d846dc277f45e Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:56:28 +0200 Subject: [PATCH 012/165] Fix Intel HEX record parsing and CS:IP handling --- .../FileFormat.FirmwareHex/IntelHexReader.cs | 53 +++++++++++++++---- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs b/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs index b90602dcc..590ec2ab6 100644 --- a/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs +++ b/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs @@ -17,6 +17,7 @@ public static FirmwareImage Read(string text) { var segments = new SortedDictionary(); uint extendedBase = 0; uint? startAddr = null; + (ushort CodeSegment, ushort InstructionPointer)? startSegmentAddress = null; var recordCount = 0; var sawEof = false; @@ -24,6 +25,8 @@ public static FirmwareImage Read(string text) { for (var line = reader.ReadLine(); line != null; line = reader.ReadLine()) { line = line.Trim(); if (line.Length == 0 || line[0] != ':') continue; + if (sawEof) + throw new InvalidDataException("IntelHex: record found after the end-of-file record."); if (line.Length < 11) throw new InvalidDataException($"IntelHex: short record '{line}'."); recordCount++; @@ -45,27 +48,38 @@ public static FirmwareImage Read(string text) { $"IntelHex: checksum mismatch (computed 0x{expected:X2}, stored 0x{bytes[^1]:X2}) on '{line}'."); var payload = bytes.AsSpan(4, len).ToArray(); + ValidateRecordShape(type, addr, payload.Length, line); switch (type) { - case 0x00: // data - AddSegment(segments, extendedBase + addr, payload); + case 0x00: { // data + var absoluteAddress = extendedBase + addr; + if ((ulong)absoluteAddress + (uint)payload.Length > 0x1_0000_0000UL) + throw new InvalidDataException($"IntelHex: data record crosses the 32-bit address-space limit in '{line}'."); + AddSegment(segments, absoluteAddress, payload); break; + } case 0x01: // EOF sawEof = true; break; case 0x02: // extended segment address (16-bit segment * 16) extendedBase = (uint)(((payload[0] << 8) | payload[1]) << 4); break; - case 0x03: // start segment address (CS:IP) - startAddr = (uint)(((payload[0] << 24) | (payload[1] << 16)) + ((payload[2] << 8) | payload[3])); + case 0x03: { // start segment address (CS:IP) + var cs = (ushort)((payload[0] << 8) | payload[1]); + var ip = (ushort)((payload[2] << 8) | payload[3]); + startAddr = ((uint)cs << 4) + ip; + startSegmentAddress = (cs, ip); break; + } case 0x04: // extended linear address (high 16 bits) extendedBase = (uint)((payload[0] << 24) | (payload[1] << 16)); break; case 0x05: // start linear address (EIP) startAddr = (uint)((payload[0] << 24) | (payload[1] << 16) | (payload[2] << 8) | payload[3]); + startSegmentAddress = null; break; default: - // Unknown record types are ignored (some vendors extend the format). + // Unknown record types are ignored after checksum validation. Some + // vendor tools extend Intel HEX beyond the six standard record types. break; } } @@ -74,7 +88,25 @@ public static FirmwareImage Read(string text) { throw new InvalidDataException("IntelHex: missing ':00000001FF' end-of-file record."); var (merged, gapCount, totalBytes) = MergeSegments(segments); - return new FirmwareImage(merged, startAddr, recordCount, gapCount, totalBytes, "IntelHex"); + return new FirmwareImage(merged, startAddr, recordCount, gapCount, totalBytes, "IntelHex") { + StartSegmentAddress = startSegmentAddress, + }; + } + + private static void ValidateRecordShape(byte type, ushort address, int payloadLength, string line) { + var expectedPayloadLength = type switch { + 0x01 => 0, + 0x02 or 0x04 => 2, + 0x03 or 0x05 => 4, + _ => -1, + }; + if (expectedPayloadLength >= 0 && payloadLength != expectedPayloadLength) + throw new InvalidDataException( + $"IntelHex: record type 0x{type:X2} requires {expectedPayloadLength} data bytes, got {payloadLength} in '{line}'."); + + if (type is 0x02 or 0x03 or 0x04 or 0x05 && address != 0) + throw new InvalidDataException( + $"IntelHex: record type 0x{type:X2} requires address 0000, got {address:X4} in '{line}'."); } private static byte[] ParseHexBytes(ReadOnlySpan hex) { @@ -90,10 +122,11 @@ private static byte[] ParseHexBytes(ReadOnlySpan hex) { } private static void AddSegment(SortedDictionary segments, uint addr, byte[] data) { - // Merge with an immediately-preceding segment so consecutive 16-byte records fold into one. + // Merge with an immediately-preceding segment so consecutive data records + // fold into one logical byte run. foreach (var k in segments.Keys) { var existing = segments[k]; - if (k + (uint)existing.Length == addr) { + if ((ulong)k + (uint)existing.Length == addr) { var merged = new byte[existing.Length + data.Length]; Buffer.BlockCopy(existing, 0, merged, 0, existing.Length); Buffer.BlockCopy(data, 0, merged, existing.Length, data.Length); @@ -107,13 +140,13 @@ private static void AddSegment(SortedDictionary segments, uint add internal static (List<(uint Address, byte[] Data)> Segments, int GapCount, int TotalBytes) MergeSegments(SortedDictionary segments) { var list = new List<(uint, byte[])>(); - uint? prevEnd = null; + ulong? prevEnd = null; var gaps = 0; var total = 0; foreach (var (addr, data) in segments) { if (prevEnd.HasValue && addr > prevEnd.Value) gaps++; list.Add((addr, data)); - prevEnd = addr + (uint)data.Length; + prevEnd = (ulong)addr + (uint)data.Length; total += data.Length; } return (list, gaps, total); From 4dd1a702be5eb41c6683be8f621fb1afefd8013e Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:57:08 +0200 Subject: [PATCH 013/165] Preserve sparse Intel HEX layout across rebuilds --- .../FirmwareHexWriter.cs | 143 ++++++++++++++++-- 1 file changed, 127 insertions(+), 16 deletions(-) diff --git a/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs b/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs index 7105155b2..77d50eb18 100644 --- a/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs +++ b/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs @@ -14,11 +14,13 @@ namespace FileFormat.FirmwareHex; /// and both are fully specified, so writing them is a transcription rather than a /// reconstruction. The reader normalises an image to a single firmware.bin /// plus a rendered metadata.ini; the writer reads that pair back — the -/// payload is the bytes, the metadata supplies the base and start addresses the -/// flat binary cannot carry. +/// payload is the bytes, the metadata supplies the address map and start address +/// information the flat binary cannot carry. /// public static class FirmwareHexWriter { + private readonly record struct SegmentLayout(uint Address, int Length); + /// The name the reader gives the flat payload. public const string PayloadName = "firmware.bin"; @@ -37,12 +39,16 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st byte[]? payload = null; var baseAddress = 0u; uint? startAddress = null; + ushort? startSegmentCs = null; + ushort? startSegmentIp = null; + var segmentLayout = new List(); foreach (var input in inputs) { if (input.IsDirectory) continue; var leaf = Path.GetFileName(input.ArchiveName); if (leaf.Equals(MetadataName, StringComparison.OrdinalIgnoreCase)) { - ReadMetadata(input.ReadContent(), ref baseAddress, ref startAddress); + ReadMetadata(input.ReadContent(), ref baseAddress, ref startAddress, + segmentLayout, ref startSegmentCs, ref startSegmentIp); continue; } // Any other single input is the payload, whatever it is called: a caller @@ -50,33 +56,130 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st payload ??= input.ReadContent(); } - var segments = payload is { Length: > 0 } - ? new List<(uint, byte[])> { (baseAddress, payload) } - : []; - return new FirmwareImage(segments, startAddress, RecordCount: 0, GapCount: 0, - TotalDataBytes: payload?.Length ?? 0, SourceFormat: sourceFormat); + List<(uint Address, byte[] Data)> segments; + if (payload is not { Length: > 0 }) + segments = []; + else if (!TryRestoreSparseSegments(payload, baseAddress, segmentLayout, out segments)) + segments = [(baseAddress, payload)]; + + var image = new FirmwareImage(segments, startAddress, RecordCount: 0, + GapCount: Math.Max(0, segments.Count - 1), + TotalDataBytes: segments.Sum(segment => segment.Data.Length), + SourceFormat: sourceFormat); + + if (startAddress is { } linear && startSegmentCs is { } cs && startSegmentIp is { } ip + && ((uint)cs << 4) + ip == linear) + image = image with { StartSegmentAddress = (cs, ip) }; + + return image; } - /// Reads the two addresses the flat payload cannot carry out of a rendered summary. - private static void ReadMetadata(byte[] metadata, ref uint baseAddress, ref uint? startAddress) { + /// + /// Reads the addresses and segment map that a flat payload cannot carry out of + /// a rendered summary. Unknown metadata is deliberately ignored so older and + /// hand-written summaries remain accepted. + /// + private static void ReadMetadata(byte[] metadata, ref uint baseAddress, ref uint? startAddress, + List segmentLayout, ref ushort? startSegmentCs, ref ushort? startSegmentIp) { foreach (var raw in Encoding.UTF8.GetString(metadata).Split('\n')) { var line = raw.Trim(); var equals = line.IndexOf('=', StringComparison.Ordinal); if (equals < 0) continue; var key = line[..equals].Trim(); var value = line[(equals + 1)..].Trim(); - if (!value.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) continue; - if (!uint.TryParse(value.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var parsed)) continue; - if (key.Equals("base_address", StringComparison.OrdinalIgnoreCase)) baseAddress = parsed; - else if (key.Equals("start_address", StringComparison.OrdinalIgnoreCase)) startAddress = parsed; + + if (key.Equals("base_address", StringComparison.OrdinalIgnoreCase)) { + if (TryParseHexUInt32(value, out var parsed)) baseAddress = parsed; + continue; + } + if (key.Equals("start_address", StringComparison.OrdinalIgnoreCase)) { + if (TryParseHexUInt32(value, out var parsed)) startAddress = parsed; + continue; + } + if (key.Equals("start_segment_cs", StringComparison.OrdinalIgnoreCase)) { + if (TryParseHexUInt16(value, out var parsed)) startSegmentCs = parsed; + continue; + } + if (key.Equals("start_segment_ip", StringComparison.OrdinalIgnoreCase)) { + if (TryParseHexUInt16(value, out var parsed)) startSegmentIp = parsed; + continue; + } + if (key.StartsWith("segment_", StringComparison.OrdinalIgnoreCase) + && TryParseSegmentLayout(value, out var segment)) + segmentLayout.Add(segment); + } + } + + private static bool TryParseHexUInt32(string text, out uint value) { + value = 0; + return text.StartsWith("0x", StringComparison.OrdinalIgnoreCase) + && uint.TryParse(text.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out value); + } + + private static bool TryParseHexUInt16(string text, out ushort value) { + value = 0; + return text.StartsWith("0x", StringComparison.OrdinalIgnoreCase) + && ushort.TryParse(text.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out value); + } + + private static bool TryParseSegmentLayout(string value, out SegmentLayout segment) { + segment = default; + const string separator = " .. "; + var separatorIndex = value.IndexOf(separator, StringComparison.Ordinal); + if (separatorIndex <= 0) return false; + + var startText = value[..separatorIndex].Trim(); + var remainder = value[(separatorIndex + separator.Length)..].Trim(); + var endTokenLength = remainder.IndexOf(' '); + if (endTokenLength < 0) endTokenLength = remainder.Length; + var endText = remainder[..endTokenLength]; + if (!TryParseHexUInt32(startText, out var start) || !TryParseHexUInt32(endText, out var end) || end < start) + return false; + + var length = (ulong)end - start; + if (length > int.MaxValue) return false; + segment = new SegmentLayout(start, (int)length); + return true; + } + + /// + /// Restores the sparse runs described by metadata from the reader's flat binary. + /// The layout is used only when it spans the payload exactly; if callers replace + /// firmware.bin with a differently sized binary, stale segment metadata is + /// ignored and the replacement becomes one contiguous run at . + /// + private static bool TryRestoreSparseSegments(byte[] payload, uint baseAddress, + IReadOnlyList layout, out List<(uint Address, byte[] Data)> segments) { + segments = []; + if (layout.Count == 0) return false; + + var ordered = layout.OrderBy(segment => segment.Address).ToArray(); + if (ordered[0].Address != baseAddress) return false; + + ulong previousEnd = baseAddress; + ulong highestEnd = baseAddress; + foreach (var segment in ordered) { + if (segment.Length < 0 || segment.Address < previousEnd) return false; + var end = (ulong)segment.Address + (uint)segment.Length; + if (end > 0x1_0000_0000UL) return false; + previousEnd = end; + highestEnd = Math.Max(highestEnd, end); } + if (highestEnd - baseAddress != (ulong)payload.Length) return false; + + foreach (var segment in ordered) { + var offset = (ulong)segment.Address - baseAddress; + if (offset + (uint)segment.Length > (ulong)payload.Length) return false; + segments.Add((segment.Address, payload.AsSpan((int)offset, segment.Length).ToArray())); + } + return true; } /// /// Writes as Intel HEX: type-04 extended-linear-address /// records whenever the high half of the address changes, type-00 data records of /// at most bytes that never straddle a 64 KiB - /// boundary, an optional type-05 start-linear-address record, and the type-01 + /// boundary, an optional type-03 or type-05 start-address record, and the type-01 /// end-of-file record every reader requires. /// public static void WriteIntelHex(Stream output, FirmwareImage image, int bytesPerRecord = 16) { @@ -88,6 +191,10 @@ public static void WriteIntelHex(Stream output, FirmwareImage image, int bytesPe var text = new StringBuilder(); var upper = -1L; foreach (var (segmentAddress, data) in image.Segments) { + if ((ulong)segmentAddress + (uint)data.Length > 0x1_0000_0000UL) + throw new InvalidDataException( + $"IntelHex: segment at 0x{segmentAddress:X8} with {data.Length} bytes exceeds the 32-bit address space."); + var offset = 0; while (offset < data.Length) { var address = segmentAddress + (uint)offset; @@ -106,8 +213,12 @@ public static void WriteIntelHex(Stream output, FirmwareImage image, int bytesPe } } - if (image.StartAddress is { } start) + if (image.StartSegmentAddress is { } segmentedStart) { + var (cs, ip) = segmentedStart; + Record(text, 0, 0x03, [(byte)(cs >> 8), (byte)cs, (byte)(ip >> 8), (byte)ip]); + } else if (image.StartAddress is { } start) { Record(text, 0, 0x05, [(byte)(start >> 24), (byte)(start >> 16), (byte)(start >> 8), (byte)start]); + } Record(text, 0, 0x01, []); var bytes = Encoding.ASCII.GetBytes(text.ToString()); From 7a27b98f22f023c03724895352340eb96cb9c150 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:57:19 +0200 Subject: [PATCH 014/165] Record Intel HEX start-address kind in metadata --- FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs b/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs index 8665c2de9..a7ea0150d 100644 --- a/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs +++ b/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs @@ -37,6 +37,14 @@ private static byte[] BuildMetadata(FirmwareImage image) { sb.Append(CultureInfo.InvariantCulture, $"base_address = 0x{image.BaseAddress:X8}\n"); sb.Append(CultureInfo.InvariantCulture, $"start_address = {(image.StartAddress.HasValue ? $"0x{image.StartAddress.Value:X8}" : "(unspecified)")}\n"); + if (image.StartSegmentAddress is { } segmentedStart) { + var (cs, ip) = segmentedStart; + sb.AppendLine("start_address_kind = segment"); + sb.Append(CultureInfo.InvariantCulture, $"start_segment_cs = 0x{cs:X4}\n"); + sb.Append(CultureInfo.InvariantCulture, $"start_segment_ip = 0x{ip:X4}\n"); + } else if (image.StartAddress.HasValue) { + sb.AppendLine("start_address_kind = linear"); + } for (var i = 0; i < image.Segments.Count; i++) { var (a, d) = image.Segments[i]; sb.Append(CultureInfo.InvariantCulture, From c0128f922ba327fe3ee6803f12f21e2a8284cc06 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:57:33 +0200 Subject: [PATCH 015/165] Promote Intel HEX to rebuild-based read-write support --- .../IntelHexFormatDescriptor.cs | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs b/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs index 460f3a92a..4622631df 100644 --- a/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs +++ b/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs @@ -9,15 +9,17 @@ namespace FileFormat.FirmwareHex; /// /// Pseudo-archive descriptor for Intel HEX firmware files. Decodes the ASCII /// records into a flat binary (firmware.bin) and surfaces a -/// metadata.ini with record count, declared start address, and gap count. +/// metadata.ini with record count, declared start address, and sparse +/// segment layout. /// /// References: /// /// Intel "Hexadecimal Object File Format Specification", Rev. A (1988) — the defining document -/// https://en.wikipedia.org/wiki/Intel_HEX — record types and checksum rules +/// https://developerhelp.microchip.com/xwiki/bin/view/software-tools/ipe/sqtp-file-format-specification/intel-hex/ — record layout and checksum rules /// /// -public sealed class IntelHexFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable { +public sealed class IntelHexFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, + IArchiveModifiable, IArchiveDefragmentable { /// /// Gets the id. @@ -36,7 +38,7 @@ public sealed class IntelHexFormatDescriptor : IFormatDescriptor, IArchiveFormat /// public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | - FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; + FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; /// /// Gets the default extension. /// @@ -95,14 +97,20 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? /// /// Writes a fresh Intel HEX file: the single payload input becomes the data /// records, and a metadata.ini alongside it -- the one this descriptor's - /// own reader renders -- supplies the base and start addresses that a flat - /// binary cannot carry. + /// own reader renders -- supplies the sparse segment map and start-address form + /// that a flat binary cannot carry. /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { ArgumentNullException.ThrowIfNull(output); FirmwareHexWriter.WriteIntelHex(output, FirmwareHexWriter.ImageFrom(inputs, "IntelHex")); } + // IArchiveModifiable and IArchiveDefragmentable intentionally use their shared + // verified staged-rebuild implementations. Rebuilding is the native edit model + // for a line-oriented HEX file: there are no allocation structures to patch in + // place, and the rendered metadata now preserves sparse address runs and the + // distinction between type-03 CS:IP and type-05 linear start records. + private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) { using var reader = new StreamReader(stream, Encoding.ASCII, detectEncodingFromByteOrderMarks: true, leaveOpen: true); var text = reader.ReadToEnd(); From 0489d4ea3c38d5619fa4f20a5152b5c2d405a5f9 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:57:57 +0200 Subject: [PATCH 016/165] Fix NRG writer and enable rebuild maintenance --- .../FileFormat.Nrg/NrgFormatDescriptor.cs | 183 +++++++++--------- 1 file changed, 87 insertions(+), 96 deletions(-) diff --git a/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs b/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs index 25da12c5e..d624eee6e 100644 --- a/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs +++ b/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs @@ -1,148 +1,139 @@ #pragma warning disable CS1591 +using System.Buffers.Binary; using Compression.Registry; using static Compression.Registry.FormatHelpers; namespace FileFormat.Nrg; /// -/// Nero Burning ROM NRG disc image — trailing NER5/NERO footer pointing at a chunked session/track descriptor area. +/// Nero Burning ROM NRG disc image — a sector stream followed by a chunked +/// session/track descriptor and a trailing NERO/NER5 footer. /// /// References: /// -/// https://cdemu.sourceforge.io — CDEmu / libMirage — its NRG parser is the de-facto format documentation -/// https://en.wikipedia.org/wiki/NRG_(file_format) — Wikipedia -/// No official specification — proprietary Nero format, reverse-engineered +/// https://cdemu.sourceforge.io — CDEmu / libMirage NRG parser, used as a behavioural oracle for the reverse-engineered chunk layout +/// https://en.wikipedia.org/wiki/NRG_(file_format) — format overview +/// No official public Nero specification is available; the format is proprietary and reverse-engineered /// /// -public sealed class NrgFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable { +public sealed class NrgFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, + IArchiveModifiable, IArchiveDefragmentable, IArchiveShrinkable { + private const uint CdRomMediumType = 0x00000400; - /// - /// Gets the id. - /// + /// Gets the id. public string Id => "Nrg"; - /// - /// Gets the display name. - /// + + /// Gets the display name. public string DisplayName => "NRG"; - /// - /// Gets the category. - /// + + /// Gets the category. public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// + + /// Gets the capabilities. public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; - /// - /// Gets the default extension. - /// + + /// Gets the default extension. public string DefaultExtension => ".nrg"; - /// - /// Gets the extensions. - /// + + /// Gets the extensions. public IReadOnlyList Extensions => [".nrg"]; - /// - /// Gets the compound extensions. - /// + + /// Gets the compound extensions. public IReadOnlyList CompoundExtensions => []; + // NRG magic is a footer signature ("NER5" or "NERO" at a variable offset from EOF), // which cannot be represented as a fixed-offset MagicSignature. - // Detection relies on the file extension and footer heuristic in NrgReader. - /// - /// Gets the magic signatures. - /// + /// Gets the magic signatures. public IReadOnlyList MagicSignatures => []; - /// - /// Gets the methods. - /// + + /// Gets the methods. public IReadOnlyList Methods => [new("iso9660", "ISO 9660")]; - /// - /// Gets the tar compression format id. - /// + + /// Gets the tar compression format id. public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// + + /// Gets the family. public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// - public string Description => "Nero Burning ROM disc image (R/W via in-place sector rewrite at fixed offsets; inner ISO 9660 directory mutation delegated to FileSystem.Iso; multi-track DAOI/CUEX layouts deferred)"; - /// - /// Lists the entries in the supplied container. - /// + /// Gets the description. + public string Description => + "Nero Burning ROM disc image (NRG v1/v2 reader; interoperable v2 TAO ISO writer; R/W and maintenance through verified rebuild)"; + + /// Lists the entries in the supplied container. public List List(Stream stream, string? password) { - var r = new NrgReader(stream, leaveOpen: true); - return r.Entries.Select((e, i) => new ArchiveEntryInfo(i, e.FullPath, e.Size, - e.Size, "iso9660", e.IsDirectory, false, null)).ToList(); + using var reader = new NrgReader(stream, leaveOpen: true); + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo(index, entry.FullPath, entry.Size, + entry.Size, "iso9660", entry.IsDirectory, false, null)).ToList(); } - /// - /// Decodes the supplied input. - /// + /// Decodes the supplied input. public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new NrgReader(stream, leaveOpen: true); - foreach (var e in r.Entries) { - if (e.IsDirectory) continue; - if (files != null && !MatchesFilter(e.FullPath, files)) continue; - WriteFile(outputDir, e.FullPath, r.Extract(e)); + using var reader = new NrgReader(stream, leaveOpen: true); + foreach (var entry in reader.Entries) { + if (entry.IsDirectory) + continue; + if (files != null && !MatchesFilter(entry.FullPath, files)) + continue; + WriteFile(outputDir, entry.FullPath, reader.Extract(entry)); } } /// - /// Performs the create operation. + /// Creates a single-session, single-track NRG v2 image containing a cooked + /// 2,048-byte/sector ISO 9660 track. The trailer is a real NRG descriptor + /// chain rather than merely a NER5 signature: ETN2 describes the TAO track, + /// SINF records the session track count, MTYP marks CD-ROM media, and END! + /// terminates the chunk list. /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { - // WORM: ISO 9660 image followed by an NRG v2 ("NER5") footer. The reader - // only uses the footer to determine version; the chunk-table offset isn't - // dereferenced for ISO extraction, so we point it at end-of-file. + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + var iso = new FileSystem.Iso.IsoWriter(); - foreach (var (name, data) in FlatFiles(inputs)) - iso.AddFile(name, data); + foreach (var input in inputs.Where(static input => !input.IsDirectory)) + iso.AddFile(input.ArchiveName.Replace('\\', '/'), input.ReadContent()); + var image = iso.Build(); output.Write(image); - // Footer: "NER5" (4 bytes) + uint64 BE chunk-table offset. + + var trailerOffset = checked((ulong)output.Position); + + Span etn2 = stackalloc byte[32]; + BinaryPrimitives.WriteUInt64BigEndian(etn2, 0); // track starts at file offset 0 + BinaryPrimitives.WriteUInt64BigEndian(etn2[8..], checked((ulong)image.LongLength)); + etn2[19] = 0x00; // Mode 1, cooked 2,048-byte user-data sectors + BinaryPrimitives.WriteUInt32BigEndian(etn2[20..], 0); // first logical sector + WriteChunk(output, "ETN2"u8, etn2); + + Span sinf = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(sinf, 1); // one track in this session + WriteChunk(output, "SINF"u8, sinf); + + Span mtyp = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(mtyp, CdRomMediumType); + WriteChunk(output, "MTYP"u8, mtyp); + + WriteChunk(output, "END!"u8, []); + Span footer = stackalloc byte[12]; - footer[0] = (byte)'N'; footer[1] = (byte)'E'; footer[2] = (byte)'R'; footer[3] = (byte)'5'; - System.Buffers.Binary.BinaryPrimitives.WriteUInt64BigEndian(footer[4..], (ulong)image.Length); + "NER5"u8.CopyTo(footer); + BinaryPrimitives.WriteUInt64BigEndian(footer[4..], trailerOffset); output.Write(footer); } - // ── IArchiveModifiable ────────────────────────────────────────────── + private static void WriteChunk(Stream output, ReadOnlySpan id, ReadOnlySpan payload) { + if (id.Length != 4) + throw new ArgumentException("NRG chunk ids are exactly four bytes.", nameof(id)); - /// - /// Rewrites raw CD sectors in place. Inputs whose ArchiveName matches - /// sector-NNNNNN.bin are written at the fixed byte offset - /// lba * sectorSize + dataOffset; everything outside the touched - /// 2 048-byte user-data region — including the trailing NRG footer — stays - /// byte-identical (the footer migrates with the new EOF when the data area - /// grows past the previous end). - /// - /// Inputs not matching the synthetic sector schema are skipped — - /// inner-ISO 9660 directory mutation is delegated to FileSystem.Iso - /// and is out of scope for the sector-rewrite modifier. - /// - public void Add(Stream archive, IReadOnlyList inputs) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(inputs); - NrgInPlaceModifier.AddOrReplaceSectors(archive, - inputs.Where(i => !i.IsDirectory).Select(i => (i.ArchiveName, i.ReadContent()))); - } - - /// - /// Zeros the 2 048-byte user-data region of each named sector. Sector - /// framing bytes (sync / address / mode / EDC) on raw geometries and the - /// trailing NRG footer are preserved so the LBA-to-offset map and the rest - /// of the image remain byte-identical. - /// - public void Remove(Stream archive, string[] entryNames) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(entryNames); - NrgInPlaceModifier.RemoveSectors(archive, entryNames); + Span header = stackalloc byte[8]; + id.CopyTo(header); + BinaryPrimitives.WriteUInt32BigEndian(header[4..], checked((uint)payload.Length)); + output.Write(header); + output.Write(payload); } } From 8e2d731905fd82a1e926841ca676d0bb21e25e5e Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:58:31 +0200 Subject: [PATCH 017/165] Implement real Intel HEX purge --- .../IntelHexFormatDescriptor.cs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs b/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs index 4622631df..8f5f929bf 100644 --- a/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs +++ b/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs @@ -105,6 +105,31 @@ public void Create(Stream output, IReadOnlyList inputs, Format FirmwareHexWriter.WriteIntelHex(output, FirmwareHexWriter.ImageFrom(inputs, "IntelHex")); } + /// + /// Removes all programmed bytes and start-address state while leaving the + /// canonical valid empty Intel HEX document (the EOF record). This overrides + /// the generic pseudo-archive purge because firmware.bin is a rendered + /// view that also exists as a zero-length view of an empty image. + /// + public void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("Intel HEX purge requires a readable, writable, seekable stream.", nameof(archive)); + + archive.Position = 0; + _ = BuildEntries(archive); // validate before committing a destructive edit + + using var rebuilt = new MemoryStream(); + FirmwareHexWriter.WriteIntelHex(rebuilt, + new FirmwareImage([], StartAddress: null, RecordCount: 0, GapCount: 0, TotalDataBytes: 0, SourceFormat: "IntelHex")); + archive.Position = 0; + archive.SetLength(0); + rebuilt.Position = 0; + rebuilt.CopyTo(archive); + archive.Flush(); + archive.Position = 0; + } + // IArchiveModifiable and IArchiveDefragmentable intentionally use their shared // verified staged-rebuild implementations. Rebuilding is the native edit model // for a line-oriented HEX file: there are no allocation structures to patch in From 887bcc77c381305847418ebd9fa54bf7b02f0b22 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:58:39 +0200 Subject: [PATCH 018/165] Add Lynx directory layout boundary --- FileFormats/FileFormat.T64/LynxReader.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/FileFormats/FileFormat.T64/LynxReader.cs b/FileFormats/FileFormat.T64/LynxReader.cs index 4304f835b..8623f6561 100644 --- a/FileFormats/FileFormat.T64/LynxReader.cs +++ b/FileFormats/FileFormat.T64/LynxReader.cs @@ -27,6 +27,7 @@ internal sealed class LynxReader { public int FileCount { get; private set; } public string Signature { get; private set; } = string.Empty; public byte[] BasicHeader { get; private set; } = []; + public long DirectoryContentEnd { get; private set; } public long DataStart => (long)this.DirectoryBlocks * BlockSize; public long LogicalDataEnd { get; private set; } @@ -141,6 +142,7 @@ private void Parse() { throw new InvalidDataException($"Lynx entry '{name}' allocation extends beyond the archive."); } + this.DirectoryContentEnd = cursor; this.LogicalDataEnd = allocationOffset; } From 30e925ad2cf001bfff9f9359ba5828cda3534de2 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:58:54 +0200 Subject: [PATCH 019/165] Parse NRG track descriptors and offsets --- FileFormats/FileFormat.Nrg/NrgReader.cs | 435 ++++++++++++++---------- 1 file changed, 257 insertions(+), 178 deletions(-) diff --git a/FileFormats/FileFormat.Nrg/NrgReader.cs b/FileFormats/FileFormat.Nrg/NrgReader.cs index 9278ef0bb..d31747853 100644 --- a/FileFormats/FileFormat.Nrg/NrgReader.cs +++ b/FileFormats/FileFormat.Nrg/NrgReader.cs @@ -1,43 +1,32 @@ +using System.Buffers.Binary; using System.Text; namespace FileFormat.Nrg; /// -/// Reads the ISO 9660 file system embedded in a Nero Burning ROM NRG disc image. -/// NRG images carry a footer at the end of the file identifying the format version -/// and providing a chunk table that describes the track layout. -/// -/// Footer layout: -/// -/// NRG v2: last 12 bytes — "NER5" (4 bytes) + uint64 BE offset to chunk table. -/// NRG v1: last 8 bytes — "NERO" (4 bytes) + uint32 BE offset to chunk table. -/// -/// This reader parses the footer to locate the data area, then heuristically detects -/// the sector geometry and parses the ISO 9660 file system. -/// +/// Reads the ISO 9660 data track embedded in a Nero Burning ROM NRG image. +/// NRG stores disc sectors first, then a chunked session/track descriptor, and +/// finally a footer pointing back to that descriptor. /// public sealed class NrgReader : IDisposable { - // Footer magic - private static readonly byte[] MagicNer5 = [(byte)'N', (byte)'E', (byte)'R', (byte)'5']; - private static readonly byte[] MagicNero = [(byte)'N', (byte)'E', (byte)'R', (byte)'O']; - - // ISO 9660 constants private const int Iso9660SectorSize = 2048; private const int RawSectorSize = 2352; private const int SectorSize2336 = 2336; + private const int RawSectorWithSubchannelSize = 2448; private const int PvdLba = 16; private const int Mode1DataOffset = 16; private const int Mode2Form1DataOffset = 24; private readonly Stream _stream; private readonly bool _leaveOpen; - private bool _disposed; - private readonly int _sectorSize; private readonly int _dataOffset; + private readonly long _trackOffset; + private readonly long _dataAreaEnd; + private bool _disposed; - // Offset within the stream where the data (track) area begins - private readonly long _dataAreaOffset; + private readonly record struct FooterInfo(int Version, long TrailerOffset, long FooterOffset); + private readonly record struct TrackCandidate(long Offset, long Length, int SectorSize, int DataOffset); /// Gets the NRG format version detected from the footer (1 or 2), or 0 if no valid footer was found. public int Version { get; } @@ -45,279 +34,369 @@ public sealed class NrgReader : IDisposable { /// Gets all file and directory entries found in the ISO 9660 file system. public IReadOnlyList Entries { get; } - /// - /// Initializes a new from an NRG stream. - /// - /// The stream containing the NRG image data. - /// Whether to leave the stream open on dispose. + /// Initializes a new from an NRG stream. public NrgReader(Stream stream, bool leaveOpen = false) { this._stream = stream ?? throw new ArgumentNullException(nameof(stream)); + if (!stream.CanRead || !stream.CanSeek) + throw new ArgumentException("NRG reading requires a readable, seekable stream.", nameof(stream)); this._leaveOpen = leaveOpen; - (this.Version, this._dataAreaOffset) = ReadFooter(stream); - - // Treat the data area as a sub-stream starting at _dataAreaOffset - (this._sectorSize, this._dataOffset) = DetectSectorGeometry(stream, this._dataAreaOffset); + var footer = ReadFooter(stream); + this.Version = footer.Version; + + if (TryFindIsoTrack(stream, footer, out var track)) { + this._trackOffset = track.Offset; + this._sectorSize = track.SectorSize; + this._dataOffset = track.DataOffset; + this._dataAreaEnd = TrackEnd(track, footer.FooterOffset > 0 ? footer.FooterOffset : stream.Length); + } else { + var end = footer.Version != 0 ? footer.TrailerOffset : stream.Length; + (this._sectorSize, this._dataOffset) = DetectSectorGeometry(stream, 0, end); + this._trackOffset = 0; + this._dataAreaEnd = end; + } var entries = new List(); TryParseIso9660(entries); this.Entries = entries; } - /// - /// Extracts the raw data for a file entry. - /// - /// The file entry to extract. Must not be a directory. - /// The file data bytes. + /// Extracts the raw data for a file entry. public byte[] Extract(NrgEntry entry) { ArgumentNullException.ThrowIfNull(entry); if (entry.IsDirectory) throw new ArgumentException("Cannot extract a directory entry.", nameof(entry)); if (entry.Size == 0) return []; + if (entry.Size > int.MaxValue) + throw new NotSupportedException("NRG entries larger than 2 GiB require a streaming extraction API."); - return ReadFileData(entry.StartLba, (int)entry.Size); + return ReadFileData(entry.StartLba, checked((int)entry.Size)); } - // ------------------------------------------------------------------------- - // Footer parsing - // ------------------------------------------------------------------------- - - private static (int Version, long DataAreaOffset) ReadFooter(Stream stream) { - if (stream.Length < 12) - return (0, 0); - - // Check NRG v2: last 12 bytes = "NER5" + 8-byte BE offset - stream.Position = stream.Length - 12; - Span footer12 = stackalloc byte[12]; - if (stream.Read(footer12) == 12 && - footer12[0] == MagicNer5[0] && footer12[1] == MagicNer5[1] && - footer12[2] == MagicNer5[2] && footer12[3] == MagicNer5[3]) { - var chunkOffset = ReadUInt64BE(footer12, 4); - // Data area starts at beginning of file; chunk table is at chunkOffset - // The data area offset is 0 (data precedes the chunk table) - return (2, 0); + private static FooterInfo ReadFooter(Stream stream) { + if (stream.Length >= 12) { + stream.Position = stream.Length - 12; + Span footer = stackalloc byte[12]; + if (ReadExactly(stream, footer) && footer[..4].SequenceEqual("NER5"u8)) { + var offset = BinaryPrimitives.ReadUInt64BigEndian(footer[4..]); + var footerOffset = stream.Length - 12; + if (offset <= (ulong)footerOffset) + return new(2, checked((long)offset), footerOffset); + } } - // Check NRG v1: last 8 bytes = "NERO" + 4-byte BE offset if (stream.Length >= 8) { stream.Position = stream.Length - 8; - Span footer8 = stackalloc byte[8]; - if (stream.Read(footer8) == 8 && - footer8[0] == MagicNero[0] && footer8[1] == MagicNero[1] && - footer8[2] == MagicNero[2] && footer8[3] == MagicNero[3]) { - return (1, 0); + Span footer = stackalloc byte[8]; + if (ReadExactly(stream, footer) && footer[..4].SequenceEqual("NERO"u8)) { + var offset = BinaryPrimitives.ReadUInt32BigEndian(footer[4..]); + var footerOffset = stream.Length - 8; + if (offset <= footerOffset) + return new(1, offset, footerOffset); } } - // No recognized footer — treat entire stream as raw sector data starting at offset 0 - return (0, 0); + return new(0, stream.Length, stream.Length); } - // ------------------------------------------------------------------------- - // Sector geometry detection - // ------------------------------------------------------------------------- + private static bool TryFindIsoTrack(Stream stream, FooterInfo footer, out TrackCandidate track) { + track = default; + if (footer.Version == 0 || footer.TrailerOffset >= footer.FooterOffset) + return false; - private static (int SectorSize, int DataOffset) DetectSectorGeometry(Stream stream, long dataAreaOffset) { - if (TryProbe(stream, dataAreaOffset, RawSectorSize, Mode1DataOffset)) - return (RawSectorSize, Mode1DataOffset); + foreach (var candidate in ReadTrackCandidates(stream, footer)) { + var end = TrackEnd(candidate, footer.TrailerOffset); + if (TryProbe(stream, candidate.Offset, candidate.SectorSize, candidate.DataOffset, end)) { + track = candidate; + return true; + } + } - if (TryProbe(stream, dataAreaOffset, RawSectorSize, Mode2Form1DataOffset)) - return (RawSectorSize, Mode2Form1DataOffset); + return false; + } - if (TryProbe(stream, dataAreaOffset, SectorSize2336, 8)) - return (SectorSize2336, 8); + private static IEnumerable ReadTrackCandidates(Stream stream, FooterInfo footer) { + var position = footer.TrailerOffset; + while (position <= footer.FooterOffset - 8) { + stream.Position = position; + Span header = stackalloc byte[8]; + if (!ReadExactly(stream, header)) + yield break; + + var payloadLength = BinaryPrimitives.ReadUInt32BigEndian(header[4..]); + var payloadStart = position + 8; + if (payloadLength > (ulong)(footer.FooterOffset - payloadStart)) + yield break; + var payloadEnd = payloadStart + payloadLength; + + if (header[..4].SequenceEqual("ETN2"u8)) { + foreach (var candidate in ReadEtn2Candidates(stream, payloadStart, payloadLength)) + yield return candidate; + } else if (header[..4].SequenceEqual("ETNF"u8)) { + foreach (var candidate in ReadEtnfCandidates(stream, payloadStart, payloadLength)) + yield return candidate; + } else if (header[..4].SequenceEqual("DAOX"u8)) { + foreach (var candidate in ReadDaoCandidates(stream, payloadStart, payloadLength, isV2: true)) + yield return candidate; + } else if (header[..4].SequenceEqual("DAOI"u8)) { + foreach (var candidate in ReadDaoCandidates(stream, payloadStart, payloadLength, isV2: false)) + yield return candidate; + } - if (TryProbe(stream, dataAreaOffset, Iso9660SectorSize, 0)) - return (Iso9660SectorSize, 0); + position = payloadEnd; + if (header[..4].SequenceEqual("END!"u8)) + yield break; + } + } + + private static IEnumerable ReadEtn2Candidates(Stream stream, long payloadStart, uint payloadLength) { + const int recordSize = 32; + Span record = stackalloc byte[recordSize]; + for (long relative = 0; relative + recordSize <= payloadLength; relative += recordSize) { + stream.Position = payloadStart + relative; + if (!ReadExactly(stream, record)) + yield break; + + var offset = BinaryPrimitives.ReadUInt64BigEndian(record); + var length = BinaryPrimitives.ReadUInt64BigEndian(record[8..]); + if (offset > long.MaxValue || length > long.MaxValue) + continue; + if (TryDecodeMode(record[19], declaredSectorSize: 0, out var sectorSize, out var dataOffset)) + yield return new(checked((long)offset), checked((long)length), sectorSize, dataOffset); + } + } + + private static IEnumerable ReadEtnfCandidates(Stream stream, long payloadStart, uint payloadLength) { + const int recordSize = 20; + Span record = stackalloc byte[recordSize]; + for (long relative = 0; relative + recordSize <= payloadLength; relative += recordSize) { + stream.Position = payloadStart + relative; + if (!ReadExactly(stream, record)) + yield break; + + var offset = BinaryPrimitives.ReadUInt32BigEndian(record); + var length = BinaryPrimitives.ReadUInt32BigEndian(record[4..]); + if (TryDecodeMode(record[11], declaredSectorSize: 0, out var sectorSize, out var dataOffset)) + yield return new(offset, length, sectorSize, dataOffset); + } + } + + private static IEnumerable ReadDaoCandidates(Stream stream, long payloadStart, uint payloadLength, bool isV2) { + const int daoHeaderSize = 22; + var recordSize = isV2 ? 42 : 30; + if (payloadLength < daoHeaderSize) + yield break; + + var record = new byte[recordSize]; + for (long relative = daoHeaderSize; relative + recordSize <= payloadLength; relative += recordSize) { + stream.Position = payloadStart + relative; + if (!ReadExactly(stream, record)) + yield break; + + var declaredSectorSize = BinaryPrimitives.ReadUInt16BigEndian(record.AsSpan(12, 2)); + var mode = record[14]; + ulong start; + ulong end; + if (isV2) { + start = BinaryPrimitives.ReadUInt64BigEndian(record.AsSpan(26, 8)); + end = BinaryPrimitives.ReadUInt64BigEndian(record.AsSpan(34, 8)); + } else { + start = BinaryPrimitives.ReadUInt32BigEndian(record.AsSpan(22, 4)); + end = BinaryPrimitives.ReadUInt32BigEndian(record.AsSpan(26, 4)); + } + + if (start > long.MaxValue || end > long.MaxValue || end < start) + continue; + if (TryDecodeMode(mode, declaredSectorSize, out var sectorSize, out var dataOffset)) + yield return new(checked((long)start), checked((long)(end - start)), sectorSize, dataOffset); + } + } + private static bool TryDecodeMode(byte mode, int declaredSectorSize, out int sectorSize, out int dataOffset) { + (sectorSize, dataOffset) = mode switch { + 0x00 or 0x02 => (Iso9660SectorSize, 0), + 0x03 => (SectorSize2336, 0), + 0x05 => (RawSectorSize, Mode1DataOffset), + 0x06 => (RawSectorSize, Mode2Form1DataOffset), + 0x0F => (RawSectorWithSubchannelSize, Mode1DataOffset), + 0x11 => (RawSectorWithSubchannelSize, Mode2Form1DataOffset), + _ => declaredSectorSize switch { + Iso9660SectorSize => (Iso9660SectorSize, 0), + _ => (0, 0), + }, + }; + return sectorSize != 0; + } + + private static long TrackEnd(TrackCandidate candidate, long hardEnd) { + if (candidate.Offset < 0 || candidate.Offset >= hardEnd) + return candidate.Offset; + if (candidate.Length <= 0) + return hardEnd; + var remaining = hardEnd - candidate.Offset; + return candidate.Offset + Math.Min(candidate.Length, remaining); + } + + private static (int SectorSize, int DataOffset) DetectSectorGeometry(Stream stream, long trackOffset, long dataEnd) { + if (TryProbe(stream, trackOffset, RawSectorSize, Mode1DataOffset, dataEnd)) + return (RawSectorSize, Mode1DataOffset); + if (TryProbe(stream, trackOffset, RawSectorSize, Mode2Form1DataOffset, dataEnd)) + return (RawSectorSize, Mode2Form1DataOffset); + if (TryProbe(stream, trackOffset, SectorSize2336, 8, dataEnd)) + return (SectorSize2336, 8); + if (TryProbe(stream, trackOffset, Iso9660SectorSize, 0, dataEnd)) + return (Iso9660SectorSize, 0); return (RawSectorSize, Mode1DataOffset); } - private static bool TryProbe(Stream stream, long dataAreaOffset, int sectorSize, int dataOffset) { - var pvdPos = dataAreaOffset + (long)PvdLba * sectorSize + dataOffset; - if (pvdPos + 6 > stream.Length) + private static bool TryProbe(Stream stream, long trackOffset, int sectorSize, int dataOffset, long dataEnd) { + if (trackOffset < 0 || sectorSize <= 0 || dataOffset < 0) return false; - - Span sig = stackalloc byte[6]; - stream.Position = pvdPos; - var read = stream.Read(sig); - if (read < 6) + var pvdPosition = trackOffset + (long)PvdLba * sectorSize + dataOffset; + if (pvdPosition < trackOffset || pvdPosition > dataEnd - 6) return false; - return sig[0] == 1 && - sig[1] == (byte)'C' && - sig[2] == (byte)'D' && - sig[3] == (byte)'0' && - sig[4] == (byte)'0' && - sig[5] == (byte)'1'; + Span signature = stackalloc byte[6]; + stream.Position = pvdPosition; + return ReadExactly(stream, signature) && + signature[0] == 1 && signature[1..].SequenceEqual("CD001"u8); } - // ------------------------------------------------------------------------- - // ISO 9660 parsing - // ------------------------------------------------------------------------- - private void TryParseIso9660(List entries) { var pvd = ReadSector(PvdLba); - if (pvd == null) + if (pvd == null || pvd[0] != 1 || !pvd.AsSpan(1, 5).SequenceEqual("CD001"u8)) return; - if (pvd[0] != 1 || - pvd[1] != (byte)'C' || pvd[2] != (byte)'D' || - pvd[3] != (byte)'0' || pvd[4] != (byte)'0' || pvd[5] != (byte)'1') + var rootLba = BinaryPrimitives.ReadUInt32LittleEndian(pvd.AsSpan(158, 4)); + var rootSize = BinaryPrimitives.ReadUInt32LittleEndian(pvd.AsSpan(166, 4)); + if (rootLba > int.MaxValue || rootSize > int.MaxValue) return; - var rootLba = (int)ReadUInt32LE(pvd, 156 + 2); - var rootSize = (int)ReadUInt32LE(pvd, 156 + 10); - - WalkDirectory(rootLba, rootSize, "", entries); + WalkDirectory((int)rootLba, (int)rootSize, "", entries, []); } - private void WalkDirectory(int dirLba, int dirSize, string parentPath, List entries) { - if (dirLba <= 0 || dirSize <= 0) + private void WalkDirectory(int dirLba, int dirSize, string parentPath, List entries, HashSet visited) { + if (dirLba <= 0 || dirSize <= 0 || !visited.Add(dirLba)) return; var bytesRead = 0; var currentLba = dirLba; - var bufOffset = 0; + var bufferOffset = 0; byte[]? sector = null; while (bytesRead < dirSize) { - if (sector == null || bufOffset >= Iso9660SectorSize) { + if (sector == null || bufferOffset >= Iso9660SectorSize) { sector = ReadSector(currentLba); if (sector == null) return; - currentLba++; - bufOffset = 0; + ++currentLba; + bufferOffset = 0; } - var recordLen = sector[bufOffset]; - - if (recordLen == 0) { - var remaining = Iso9660SectorSize - bufOffset; + var recordLength = sector[bufferOffset]; + if (recordLength == 0) { + var remaining = Iso9660SectorSize - bufferOffset; bytesRead += remaining; - bufOffset = Iso9660SectorSize; + bufferOffset = Iso9660SectorSize; continue; } - if (bufOffset + recordLen > Iso9660SectorSize) { - var remaining = Iso9660SectorSize - bufOffset; + if (recordLength < 34 || bufferOffset + recordLength > Iso9660SectorSize) { + var remaining = Iso9660SectorSize - bufferOffset; bytesRead += remaining; - bufOffset = Iso9660SectorSize; + bufferOffset = Iso9660SectorSize; continue; } - var record = sector.AsSpan(bufOffset, recordLen); - ParseDirectoryRecord(record, parentPath, entries); - - bytesRead += recordLen; - bufOffset += recordLen; + var record = sector.AsSpan(bufferOffset, recordLength); + ParseDirectoryRecord(record, parentPath, entries, visited); + bytesRead += recordLength; + bufferOffset += recordLength; } } - private void ParseDirectoryRecord(ReadOnlySpan record, string parentPath, List entries) { - if (record.Length < 34) + private void ParseDirectoryRecord(ReadOnlySpan record, string parentPath, List entries, HashSet visited) { + var idLength = record[32]; + if (idLength == 0 || 33 + idLength > record.Length) return; - - var dataLba = (int)ReadUInt32LE(record, 2); - var dataLen = (int)ReadUInt32LE(record, 10); - var flags = record[25]; - var idLen = record[32]; - - if (idLen == 0 || record.Length < 33 + idLen) + if (idLength == 1 && (record[33] == 0x00 || record[33] == 0x01)) return; - if (idLen == 1 && (record[33] == 0x00 || record[33] == 0x01)) + var dataLba = BinaryPrimitives.ReadUInt32LittleEndian(record[2..6]); + var dataLength = BinaryPrimitives.ReadUInt32LittleEndian(record[10..14]); + if (dataLba > int.MaxValue || dataLength > int.MaxValue) return; - var isDirectory = (flags & 0x02) != 0; - var rawName = Encoding.ASCII.GetString(record.Slice(33, idLen)); + var isDirectory = (record[25] & 0x02) != 0; + var rawName = Encoding.ASCII.GetString(record.Slice(33, idLength)); var name = StripVersionSuffix(rawName); - if (string.IsNullOrEmpty(name)) return; - var fullPath = parentPath.Length > 0 ? parentPath + "/" + name : name; - + var fullPath = parentPath.Length > 0 ? $"{parentPath}/{name}" : name; entries.Add(new NrgEntry { Name = name, FullPath = fullPath, IsDirectory = isDirectory, - Size = isDirectory ? 0 : dataLen, - StartLba = dataLba, + Size = isDirectory ? 0 : dataLength, + StartLba = (int)dataLba, }); if (isDirectory) - WalkDirectory(dataLba, dataLen, fullPath, entries); + WalkDirectory((int)dataLba, (int)dataLength, fullPath, entries, visited); } - // ------------------------------------------------------------------------- - // Data extraction - // ------------------------------------------------------------------------- - private byte[] ReadFileData(int startLba, int size) { var result = new byte[size]; var written = 0; - var lba = startLba; - - while (written < size) { + for (var lba = startLba; written < size; ++lba) { var sector = ReadSector(lba); if (sector == null) - break; + throw new InvalidDataException("NRG file extent runs past the selected data track."); var toCopy = Math.Min(Iso9660SectorSize, size - written); sector.AsSpan(0, toCopy).CopyTo(result.AsSpan(written)); written += toCopy; - lba++; } - return result; } private byte[]? ReadSector(int lba) { - var sectorStart = this._dataAreaOffset + (long)lba * this._sectorSize; + if (lba < 0) + return null; + var sectorStart = this._trackOffset + (long)lba * this._sectorSize; var dataStart = sectorStart + this._dataOffset; - - if (dataStart + Iso9660SectorSize > this._stream.Length) + if (sectorStart < this._trackOffset || dataStart < sectorStart || dataStart > this._dataAreaEnd - Iso9660SectorSize) return null; this._stream.Position = dataStart; - var buf = new byte[Iso9660SectorSize]; - var totalRead = 0; - while (totalRead < buf.Length) { - var read = this._stream.Read(buf, totalRead, buf.Length - totalRead); + var buffer = new byte[Iso9660SectorSize]; + return ReadExactly(this._stream, buffer) ? buffer : null; + } + + private static bool ReadExactly(Stream stream, Span buffer) { + var offset = 0; + while (offset < buffer.Length) { + var read = stream.Read(buffer[offset..]); if (read == 0) - return null; - totalRead += read; + return false; + offset += read; } - - return buf; + return true; } - // ------------------------------------------------------------------------- - // Helpers - // ------------------------------------------------------------------------- - - private static uint ReadUInt32LE(ReadOnlySpan data, int offset) => - (uint)(data[offset] | (data[offset + 1] << 8) | (data[offset + 2] << 16) | (data[offset + 3] << 24)); - - private static ulong ReadUInt64BE(ReadOnlySpan data, int offset) => - ((ulong)data[offset] << 56) | ((ulong)data[offset + 1] << 48) | - ((ulong)data[offset + 2] << 40) | ((ulong)data[offset + 3] << 32) | - ((ulong)data[offset + 4] << 24) | ((ulong)data[offset + 5] << 16) | - ((ulong)data[offset + 6] << 8) | data[offset + 7]; - private static string StripVersionSuffix(string name) { - var semi = name.IndexOf(';'); - return semi >= 0 ? name[..semi] : name; + var separator = name.IndexOf(';'); + return separator >= 0 ? name[..separator] : name; } /// public void Dispose() { - if (!this._disposed) { - this._disposed = true; - if (!this._leaveOpen) - this._stream.Dispose(); - } + if (this._disposed) + return; + this._disposed = true; + if (!this._leaveOpen) + this._stream.Dispose(); } } From 4d25e1f074e899fbdbbe38fdf6fa5891d793ccb1 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:59:01 +0200 Subject: [PATCH 020/165] Add Intel HEX read-write and maintenance tests --- .../FirmwareHex/IntelHexRwTests.cs | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 Compression.Tests/FirmwareHex/IntelHexRwTests.cs diff --git a/Compression.Tests/FirmwareHex/IntelHexRwTests.cs b/Compression.Tests/FirmwareHex/IntelHexRwTests.cs new file mode 100644 index 000000000..e42a81b25 --- /dev/null +++ b/Compression.Tests/FirmwareHex/IntelHexRwTests.cs @@ -0,0 +1,140 @@ +#pragma warning disable CS1591 +using System.Text; +using Compression.Registry; +using FileFormat.FirmwareHex; + +namespace Compression.Tests.FirmwareHex; + +[TestFixture] +public sealed class IntelHexRwTests { + + private const string SparseHex = + ":0410000001020304E2\n" + + ":03200000AABBCCAC\n" + + ":0400000312345678E5\n" + + ":00000001FF\n"; + + [Test, Category("HappyPath")] + public void Reader_DecodesType03AsCsShift4PlusIp_AndKeepsRegisters() { + var image = IntelHexReader.Read(":0400000312345678E5\n:00000001FF\n"); + + Assert.Multiple(() => { + Assert.That(image.StartAddress, Is.EqualTo(0x000179B8u)); + Assert.That(image.StartSegmentAddress, Is.Not.Null); + Assert.That(image.StartSegmentAddress?.CodeSegment, Is.EqualTo((ushort)0x1234)); + Assert.That(image.StartSegmentAddress?.InstructionPointer, Is.EqualTo((ushort)0x5678)); + }); + } + + [TestCase(":0100000412E9\n:00000001FF\n", TestName = "Reader_RejectsWrongExtendedLinearPayloadLength")] + [TestCase(":020001040001F8\n:00000001FF\n", TestName = "Reader_RejectsNonZeroExtendedLinearAddressField")] + [TestCase(":00000001FF\n:00000001FF\n", TestName = "Reader_RejectsRecordAfterEof")] + [Category("EdgeCase")] + public void Reader_RejectsMalformedControlRecords(string text) + => Assert.That(() => IntelHexReader.Read(text), Throws.InstanceOf()); + + [Test, Category("RoundTrip")] + public void ExtractCreateRoundTrip_PreservesSparseRunsAndSegmentedStartRecord() { + var descriptor = new IntelHexFormatDescriptor(); + var extracted = Extract(descriptor, Encoding.ASCII.GetBytes(SparseHex)); + + using var rebuilt = new MemoryStream(); + descriptor.Create(rebuilt, + extracted.Select(pair => ArchiveInputInfo.InMemory(pair.Key, pair.Value)).ToArray(), + new FormatCreateOptions()); + + var text = Encoding.ASCII.GetString(rebuilt.ToArray()); + var image = IntelHexReader.Read(text); + Assert.Multiple(() => { + Assert.That(image.Segments, Has.Count.EqualTo(2), "the address hole was flattened into programmed 0xFF bytes"); + Assert.That(image.Segments[0].Address, Is.EqualTo(0x00001000u)); + Assert.That(image.Segments[0].Data, Is.EqualTo(new byte[] { 1, 2, 3, 4 }).AsCollection); + Assert.That(image.Segments[1].Address, Is.EqualTo(0x00002000u)); + Assert.That(image.Segments[1].Data, Is.EqualTo(new byte[] { 0xAA, 0xBB, 0xCC }).AsCollection); + Assert.That(image.StartSegmentAddress?.CodeSegment, Is.EqualTo((ushort)0x1234)); + Assert.That(image.StartSegmentAddress?.InstructionPointer, Is.EqualTo((ushort)0x5678)); + Assert.That(text, Does.Contain(":0400000312345678E5"), "type-03 CS:IP was rewritten as a type-05 linear start"); + }); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_AdvertisesRw_AndCanReplaceItsFirmwareView() { + var descriptor = new IntelHexFormatDescriptor(); + Assert.Multiple(() => { + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + }); + + var original = Create(descriptor, [ArchiveInputInfo.InMemory("firmware.bin", new byte[] { 1, 2, 3, 4 })]); + using var archive = Expandable(original); + ((IArchiveModifiable)descriptor).Add(archive, + [ArchiveInputInfo.InMemory("firmware.bin", new byte[] { 9, 8, 7, 6 })]); + + var extracted = Extract(descriptor, archive.ToArray()); + Assert.That(extracted["firmware.bin"], Is.EqualTo(new byte[] { 9, 8, 7, 6 }).AsCollection); + } + + [Test, Category("RoundTrip")] + public void Defragment_RebuildsCanonicallyWithoutProgrammingSparseHole() { + var descriptor = new IntelHexFormatDescriptor(); + using var archive = Expandable(Encoding.ASCII.GetBytes(SparseHex)); + + ((IArchiveDefragmentable)descriptor).Defragment(archive); + + archive.Position = 0; + using var reader = new StreamReader(archive, Encoding.ASCII, detectEncodingFromByteOrderMarks: true, leaveOpen: true); + var image = IntelHexReader.Read(reader.ReadToEnd()); + Assert.Multiple(() => { + Assert.That(image.Segments, Has.Count.EqualTo(2)); + Assert.That(image.Segments[0].Address, Is.EqualTo(0x00001000u)); + Assert.That(image.Segments[1].Address, Is.EqualTo(0x00002000u)); + Assert.That(image.TotalDataBytes, Is.EqualTo(7)); + Assert.That(image.StartSegmentAddress, Is.Not.Null); + }); + } + + [Test, Category("HappyPath")] + public void Purge_LeavesCanonicalEmptyIntelHex() { + var descriptor = new IntelHexFormatDescriptor(); + using var archive = Expandable(Encoding.ASCII.GetBytes(SparseHex)); + + ((IArchivePurgeable)descriptor).Purge(archive); + + var text = Encoding.ASCII.GetString(archive.ToArray()); + var image = IntelHexReader.Read(text); + Assert.Multiple(() => { + Assert.That(text.Trim(), Is.EqualTo(":00000001FF")); + Assert.That(image.Segments, Is.Empty); + Assert.That(image.StartAddress, Is.Null); + Assert.That(image.StartSegmentAddress, Is.Null); + }); + } + + private static byte[] Create(IntelHexFormatDescriptor descriptor, IReadOnlyList inputs) { + using var output = new MemoryStream(); + descriptor.Create(output, inputs, new FormatCreateOptions()); + return output.ToArray(); + } + + private static MemoryStream Expandable(byte[] data) { + var result = new MemoryStream(); + result.Write(data); + result.Position = 0; + return result; + } + + private static Dictionary Extract(IntelHexFormatDescriptor descriptor, byte[] image) { + var directory = Path.Combine(Path.GetTempPath(), "cwb_ihex_" + Guid.NewGuid().ToString("N")[..10]); + Directory.CreateDirectory(directory); + try { + using var input = new MemoryStream(image, writable: false); + descriptor.Extract(input, directory, null, null); + return Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories) + .ToDictionary(path => Path.GetFileName(path), File.ReadAllBytes, StringComparer.OrdinalIgnoreCase); + } finally { + try { Directory.Delete(directory, recursive: true); } catch { /* best effort */ } + } + } +} From cd2bc7022da46a9804d4f0174f074d35995f859d Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:59:15 +0200 Subject: [PATCH 021/165] Add Lynx shrink wipe purge and exact layout --- .../FileFormat.T64/LynxFormatDescriptor.cs | 129 ++++++++++++++++-- 1 file changed, 121 insertions(+), 8 deletions(-) diff --git a/FileFormats/FileFormat.T64/LynxFormatDescriptor.cs b/FileFormats/FileFormat.T64/LynxFormatDescriptor.cs index 88fa95875..d00617987 100644 --- a/FileFormats/FileFormat.T64/LynxFormatDescriptor.cs +++ b/FileFormats/FileFormat.T64/LynxFormatDescriptor.cs @@ -16,7 +16,10 @@ public sealed class LynxFormatDescriptor : IArchiveCreatable, IArchiveModifiable, IArchiveDefragmentable, + IArchiveShrinkable, IArchiveLayoutMap, + IWipeEmpty, + IArchivePurgeable, IFormatOptionsSchema { /// @@ -76,8 +79,8 @@ public sealed class LynxFormatDescriptor : /// public string Description => "Will Corley/Ultimate Lynx Commodore archive: 254-byte sector payload blocks, PRG/SEQ/USR/DEL " + - "creation, REL read/remove support, and genuine in-place add/replace/remove by shifting only " + - "the affected directory/data block ranges. No compression or checksum exists in the format."; + "creation, REL read/remove support, genuine in-place add/replace/remove, exact layout/wipe, " + + "directory/trailer shrink, and purge. No compression or checksum exists in the format."; /// /// Gets the options schema. @@ -183,6 +186,25 @@ public void Remove(Stream archive, string[] entryNames) { LynxInPlaceModifier.Remove(archive, name); } + /// + /// Removes all live entries in one pass while preserving the input archive's BASIC preamble + /// and Lynx signature. The empty directory is emitted at its minimum one-block allocation. + /// + public void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("Lynx purge requires a readable, writable, seekable stream.", nameof(archive)); + + var reader = Open(archive); + var directory = LynxWriter.BuildDirectory([], 1, reader.BasicHeader, reader.Signature); + archive.Position = 0; + archive.SetLength(0); + archive.Write(directory); + archive.SetLength(directory.Length); + archive.Flush(); + archive.Position = 0; + } + /// /// Lynx data extents are inherently contiguous and ordered by the directory. Defragmentation /// therefore consists of validating that layout and dropping transport/trailing padding after @@ -210,18 +232,109 @@ public void Defragment(Stream archive, DefragOptions options) { } /// - /// Enumerates the layout. + /// Rebuilds the directory at its smallest whole-254-byte allocation and copies the complete + /// existing data area byte-for-byte. This preserves REL side sectors and file-type metadata, + /// while reclaiming directory blocks left behind after removals and any trailing transport data. + /// + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (ReferenceEquals(input, output)) + throw new ArgumentException("Lynx shrink requires distinct input and output streams.", nameof(output)); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("Lynx shrink input must be readable and seekable.", nameof(input)); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("Lynx shrink output must be writable and seekable.", nameof(output)); + + var reader = Open(input); + var directory = LynxWriter.BuildDirectory( + reader.Entries.Select(LynxWriter.FromEntry).ToArray(), + 1, + reader.BasicHeader, + reader.Signature); + + output.Position = 0; + output.SetLength(0); + output.Write(directory); + + input.Position = reader.DataStart; + var remaining = reader.LogicalDataEnd - reader.DataStart; + var buffer = new byte[64 * 1024]; + while (remaining > 0) { + var count = (int)Math.Min(buffer.Length, remaining); + input.ReadExactly(buffer.AsSpan(0, count)); + output.Write(buffer.AsSpan(0, count)); + remaining -= count; + } + + output.SetLength(output.Position); + output.Flush(); + output.Position = 0; + } + + /// + /// Enumerates the exact byte layout: live directory text, directory padding, REL side sectors, + /// logical file bytes, per-file block padding, and any trailer beyond the archive allocation. + /// Explicit free extents make generic forensic wiping safe for this format. /// public IEnumerable EnumerateLayout(Stream archive) { var reader = Open(archive); - yield return new DefragBlockInfo(0, reader.DataStart, DefragBlockKind.MetadataReserved, "Lynx directory"); + + if (reader.DirectoryContentEnd > 0) + yield return new DefragBlockInfo(0, reader.DirectoryContentEnd, DefragBlockKind.MetadataReserved, "Lynx directory"); + if (reader.DirectoryContentEnd < reader.DataStart) + yield return new DefragBlockInfo( + reader.DirectoryContentEnd, + reader.DataStart - reader.DirectoryContentEnd, + DefragBlockKind.Free, + "Directory padding"); + foreach (var entry in reader.Entries) { - var allocated = checked((long)entry.ArchiveBlocks * LynxReader.BlockSize); - if (allocated > 0) - yield return new DefragBlockInfo(entry.AllocationOffset, allocated, DefragBlockKind.Used, entry.Name); + if (entry.DataOffset > entry.AllocationOffset) + yield return new DefragBlockInfo( + entry.AllocationOffset, + entry.DataOffset - entry.AllocationOffset, + DefragBlockKind.MetadataReserved, + $"{entry.Name} REL side sectors"); + + if (entry.Length > 0) + yield return new DefragBlockInfo(entry.DataOffset, entry.Length, DefragBlockKind.Used, entry.Name); + + var allocationEnd = checked(entry.AllocationOffset + (long)entry.ArchiveBlocks * LynxReader.BlockSize); + var payloadEnd = checked(entry.DataOffset + entry.Length); + if (payloadEnd < allocationEnd) + yield return new DefragBlockInfo( + payloadEnd, + allocationEnd - payloadEnd, + DefragBlockKind.Free, + $"{entry.Name} block padding"); } + if (reader.LogicalDataEnd < archive.Length) - yield return new DefragBlockInfo(reader.LogicalDataEnd, archive.Length - reader.LogicalDataEnd, DefragBlockKind.Free); + yield return new DefragBlockInfo( + reader.LogicalDataEnd, + archive.Length - reader.LogicalDataEnd, + DefragBlockKind.Free, + "Trailing transport data"); + } + + /// + /// Zeros only byte ranges that the Lynx layout proves unused: directory padding, payload block + /// padding and trailing transport data. Live file bytes and REL side-sector metadata are retained. + /// + public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("Lynx wipe requires a readable, writable, seekable stream.", nameof(image)); + + var imageSize = image.Length; + var layout = this.EnumerateLayout(image).ToArray(); + return UnusedSpaceWiper.WipeDeclaredFree( + image, + layout, + imageSize, + wipeClusterTips: false, + fileSizeLookup: null); } private static LynxReader Open(Stream stream) { From 49e86da8802df25a9f86defe4611ede0b6b574e1 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:59:38 +0200 Subject: [PATCH 022/165] Harden NRG chunk parsing helpers --- FileFormats/FileFormat.Nrg/NrgReader.cs | 99 +++++++++++++------------ 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/FileFormats/FileFormat.Nrg/NrgReader.cs b/FileFormats/FileFormat.Nrg/NrgReader.cs index d31747853..41724ad30 100644 --- a/FileFormats/FileFormat.Nrg/NrgReader.cs +++ b/FileFormats/FileFormat.Nrg/NrgReader.cs @@ -48,7 +48,7 @@ public NrgReader(Stream stream, bool leaveOpen = false) { this._trackOffset = track.Offset; this._sectorSize = track.SectorSize; this._dataOffset = track.DataOffset; - this._dataAreaEnd = TrackEnd(track, footer.FooterOffset > 0 ? footer.FooterOffset : stream.Length); + this._dataAreaEnd = TrackEnd(track, footer.TrailerOffset); } else { var end = footer.Version != 0 ? footer.TrailerOffset : stream.Length; (this._sectorSize, this._dataOffset) = DetectSectorGeometry(stream, 0, end); @@ -116,100 +116,103 @@ private static bool TryFindIsoTrack(Stream stream, FooterInfo footer, out TrackC return false; } - private static IEnumerable ReadTrackCandidates(Stream stream, FooterInfo footer) { + private static List ReadTrackCandidates(Stream stream, FooterInfo footer) { + var result = new List(); var position = footer.TrailerOffset; - while (position <= footer.FooterOffset - 8) { + var header = new byte[8]; + + while (position <= footer.FooterOffset - header.Length) { stream.Position = position; - Span header = stackalloc byte[8]; if (!ReadExactly(stream, header)) - yield break; + break; - var payloadLength = BinaryPrimitives.ReadUInt32BigEndian(header[4..]); - var payloadStart = position + 8; - if (payloadLength > (ulong)(footer.FooterOffset - payloadStart)) - yield break; + var headerSpan = header.AsSpan(); + var payloadLength = BinaryPrimitives.ReadUInt32BigEndian(headerSpan[4..]); + var payloadStart = position + header.Length; + if (payloadStart > footer.FooterOffset || payloadLength > (ulong)(footer.FooterOffset - payloadStart)) + break; var payloadEnd = payloadStart + payloadLength; - if (header[..4].SequenceEqual("ETN2"u8)) { - foreach (var candidate in ReadEtn2Candidates(stream, payloadStart, payloadLength)) - yield return candidate; - } else if (header[..4].SequenceEqual("ETNF"u8)) { - foreach (var candidate in ReadEtnfCandidates(stream, payloadStart, payloadLength)) - yield return candidate; - } else if (header[..4].SequenceEqual("DAOX"u8)) { - foreach (var candidate in ReadDaoCandidates(stream, payloadStart, payloadLength, isV2: true)) - yield return candidate; - } else if (header[..4].SequenceEqual("DAOI"u8)) { - foreach (var candidate in ReadDaoCandidates(stream, payloadStart, payloadLength, isV2: false)) - yield return candidate; - } + if (headerSpan[..4].SequenceEqual("ETN2"u8)) + ReadEtn2Candidates(stream, payloadStart, payloadLength, result); + else if (headerSpan[..4].SequenceEqual("ETNF"u8)) + ReadEtnfCandidates(stream, payloadStart, payloadLength, result); + else if (headerSpan[..4].SequenceEqual("DAOX"u8)) + ReadDaoCandidates(stream, payloadStart, payloadLength, isV2: true, result); + else if (headerSpan[..4].SequenceEqual("DAOI"u8)) + ReadDaoCandidates(stream, payloadStart, payloadLength, isV2: false, result); position = payloadEnd; - if (header[..4].SequenceEqual("END!"u8)) - yield break; + if (headerSpan[..4].SequenceEqual("END!"u8)) + break; } + + return result; } - private static IEnumerable ReadEtn2Candidates(Stream stream, long payloadStart, uint payloadLength) { + private static void ReadEtn2Candidates(Stream stream, long payloadStart, uint payloadLength, List result) { const int recordSize = 32; - Span record = stackalloc byte[recordSize]; + var record = new byte[recordSize]; for (long relative = 0; relative + recordSize <= payloadLength; relative += recordSize) { stream.Position = payloadStart + relative; if (!ReadExactly(stream, record)) - yield break; + return; - var offset = BinaryPrimitives.ReadUInt64BigEndian(record); - var length = BinaryPrimitives.ReadUInt64BigEndian(record[8..]); + var span = record.AsSpan(); + var offset = BinaryPrimitives.ReadUInt64BigEndian(span); + var length = BinaryPrimitives.ReadUInt64BigEndian(span[8..]); if (offset > long.MaxValue || length > long.MaxValue) continue; - if (TryDecodeMode(record[19], declaredSectorSize: 0, out var sectorSize, out var dataOffset)) - yield return new(checked((long)offset), checked((long)length), sectorSize, dataOffset); + if (TryDecodeMode(span[19], declaredSectorSize: 0, out var sectorSize, out var dataOffset)) + result.Add(new(checked((long)offset), checked((long)length), sectorSize, dataOffset)); } } - private static IEnumerable ReadEtnfCandidates(Stream stream, long payloadStart, uint payloadLength) { + private static void ReadEtnfCandidates(Stream stream, long payloadStart, uint payloadLength, List result) { const int recordSize = 20; - Span record = stackalloc byte[recordSize]; + var record = new byte[recordSize]; for (long relative = 0; relative + recordSize <= payloadLength; relative += recordSize) { stream.Position = payloadStart + relative; if (!ReadExactly(stream, record)) - yield break; + return; - var offset = BinaryPrimitives.ReadUInt32BigEndian(record); - var length = BinaryPrimitives.ReadUInt32BigEndian(record[4..]); - if (TryDecodeMode(record[11], declaredSectorSize: 0, out var sectorSize, out var dataOffset)) - yield return new(offset, length, sectorSize, dataOffset); + var span = record.AsSpan(); + var offset = BinaryPrimitives.ReadUInt32BigEndian(span); + var length = BinaryPrimitives.ReadUInt32BigEndian(span[4..]); + if (TryDecodeMode(span[11], declaredSectorSize: 0, out var sectorSize, out var dataOffset)) + result.Add(new(offset, length, sectorSize, dataOffset)); } } - private static IEnumerable ReadDaoCandidates(Stream stream, long payloadStart, uint payloadLength, bool isV2) { + private static void ReadDaoCandidates(Stream stream, long payloadStart, uint payloadLength, bool isV2, List result) { const int daoHeaderSize = 22; var recordSize = isV2 ? 42 : 30; if (payloadLength < daoHeaderSize) - yield break; + return; var record = new byte[recordSize]; for (long relative = daoHeaderSize; relative + recordSize <= payloadLength; relative += recordSize) { stream.Position = payloadStart + relative; if (!ReadExactly(stream, record)) - yield break; + return; - var declaredSectorSize = BinaryPrimitives.ReadUInt16BigEndian(record.AsSpan(12, 2)); - var mode = record[14]; + var span = record.AsSpan(); + var declaredSectorSize = BinaryPrimitives.ReadUInt16BigEndian(span.Slice(12, 2)); + var mode = span[14]; ulong start; ulong end; if (isV2) { - start = BinaryPrimitives.ReadUInt64BigEndian(record.AsSpan(26, 8)); - end = BinaryPrimitives.ReadUInt64BigEndian(record.AsSpan(34, 8)); + start = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(26, 8)); + end = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(34, 8)); } else { - start = BinaryPrimitives.ReadUInt32BigEndian(record.AsSpan(22, 4)); - end = BinaryPrimitives.ReadUInt32BigEndian(record.AsSpan(26, 4)); + start = BinaryPrimitives.ReadUInt32BigEndian(span.Slice(22, 4)); + end = BinaryPrimitives.ReadUInt32BigEndian(span.Slice(26, 4)); } if (start > long.MaxValue || end > long.MaxValue || end < start) continue; if (TryDecodeMode(mode, declaredSectorSize, out var sectorSize, out var dataOffset)) - yield return new(checked((long)start), checked((long)(end - start)), sectorSize, dataOffset); + result.Add(new(checked((long)start), checked((long)(end - start)), sectorSize, dataOffset)); } } From 6af1c725fa0eb4e141b6858ab9b66897a38e5571 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 06:59:56 +0200 Subject: [PATCH 023/165] Cover Lynx maintenance operations --- Compression.Tests/Lynx/LynxTests.cs | 95 ++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/Compression.Tests/Lynx/LynxTests.cs b/Compression.Tests/Lynx/LynxTests.cs index 455e3e3ff..7eb4825de 100644 --- a/Compression.Tests/Lynx/LynxTests.cs +++ b/Compression.Tests/Lynx/LynxTests.cs @@ -23,6 +23,9 @@ public void Descriptor_CreatesCanonicalMultiFileArchive_AndRoundTripsBytes() { Assert.That(Encoding.ASCII.GetString(bytes, 60, 4), Is.EqualTo("LYNX")); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); }); archive.Position = 0; @@ -103,6 +106,25 @@ public void Descriptor_SupportsEmptyArchive_AndPurgeToZeroEntries() { Assert.That(archive.Length, Is.EqualTo(ReadDirectoryBlockCount(archive.ToArray()) * 254L)); } + [Test] + public void Purge_RebuildsMinimalValidEmptyArchive() { + var descriptor = new LynxFormatDescriptor(); + using var archive = Create(descriptor, [ + ArchiveInputInfo.InMemory("FIRST.PRG", Enumerable.Repeat((byte)0x11, 300).ToArray()), + ArchiveInputInfo.InMemory("SECOND.PRG", Enumerable.Repeat((byte)0x22, 400).ToArray()), + ]); + + descriptor.Purge(archive); + + Assert.Multiple(() => { + Assert.That(archive.Length, Is.EqualTo(254)); + Assert.That(ReadDirectoryBlockCount(archive.ToArray()), Is.EqualTo(1)); + }); + archive.Position = 0; + Assert.That(descriptor.List(archive, null), Is.Empty); + Assert.That(Encoding.ASCII.GetString(archive.ToArray(), 60, 4), Is.EqualTo("LYNX")); + } + [Test] public void Writer_UsesRequestedCommodoreType_AndRejectsLossyNamesOrEncryption() { var descriptor = new LynxFormatDescriptor(); @@ -119,8 +141,8 @@ public void Writer_UsesRequestedCommodoreType_AndRejectsLossyNamesOrEncryption() ArchiveInputInfo.InMemory("abcdefghijklmnop-A", Array.Empty()), ArchiveInputInfo.InMemory("abcdefghijklmnop-B", new byte[] { 1 }), ])); - Assert.Throws(() => Create(descriptor, - [ArchiveInputInfo.InMemory("MÄDCHEN.PRG", Array.Empty())])); + Assert.Throws(() => Create(descriptor, [ + ArchiveInputInfo.InMemory("MÄDCHEN.PRG", Array.Empty())])); Assert.Throws(() => Create(descriptor, [ArchiveInputInfo.InMemory("A.PRG", Array.Empty())], new FormatCreateOptions { Password = "secret" })); @@ -145,6 +167,75 @@ public void Defragment_TrimsOnlyTrailingTransportPadding() { Assert.That(descriptor.ExtractEntryToMemory(archive, "A.PRG", null), Is.EqualTo(Enumerable.Repeat((byte)1, 300).ToArray())); } + [Test] + public void Shrink_ReclaimsDirectoryOverallocationAndTrailer_WithoutChangingPayload() { + var descriptor = new LynxFormatDescriptor(); + var payload = Enumerable.Range(0, 700).Select(index => (byte)(index * 13)).ToArray(); + using var archive = Create(descriptor, [ArchiveInputInfo.InMemory("KEEP.PRG", payload)]); + var transientNames = new List(); + + for (var i = 0; i < 12; ++i) { + var name = $"TEMP{i:D2}.PRG"; + transientNames.Add(name); + descriptor.Add(archive, [ArchiveInputInfo.InMemory(name, Enumerable.Repeat((byte)(i + 1), 20 + i).ToArray())]); + } + + descriptor.Remove(archive, transientNames.ToArray()); + Assert.That(ReadDirectoryBlockCount(archive.ToArray()), Is.GreaterThan(1)); + archive.Position = archive.Length; + archive.Write(Enumerable.Repeat((byte)0xEE, 113).ToArray()); + var bloatedLength = archive.Length; + + using var shrunk = new MemoryStream(); + descriptor.Shrink(archive, shrunk); + + Assert.Multiple(() => { + Assert.That(shrunk.Length, Is.LessThan(bloatedLength)); + Assert.That(shrunk.Length % LynxReader.BlockSize, Is.Zero); + Assert.That(ReadDirectoryBlockCount(shrunk.ToArray()), Is.EqualTo(1)); + }); + shrunk.Position = 0; + Assert.That(descriptor.List(shrunk, null).Select(entry => entry.Name), Is.EqualTo(new[] { "KEEP.PRG" })); + shrunk.Position = 0; + Assert.That(descriptor.ExtractEntryToMemory(shrunk, "KEEP.PRG", null), Is.EqualTo(payload)); + } + + [Test] + public void LayoutAndWipe_ExposeAndClearOnlyUnusedBytes() { + var descriptor = new LynxFormatDescriptor(); + var payload = Enumerable.Range(0, 300).Select(index => (byte)(index * 7 + 3)).ToArray(); + using var archive = Create(descriptor, [ArchiveInputInfo.InMemory("DATA.PRG", payload)]); + archive.Position = archive.Length; + archive.Write(Enumerable.Repeat((byte)0xCC, 37).ToArray()); + + archive.Position = 0; + var free = descriptor.EnumerateLayout(archive) + .Where(extent => extent.Kind == DefragBlockKind.Free) + .ToArray(); + Assert.That(free, Is.Not.Empty); + Assert.That(free.Any(extent => extent.FileName == "Directory padding"), Is.True); + Assert.That(free.Any(extent => extent.FileName == "DATA.PRG block padding"), Is.True); + Assert.That(free.Any(extent => extent.FileName == "Trailing transport data"), Is.True); + + foreach (var extent in free) { + archive.Position = extent.Offset; + archive.Write(Enumerable.Repeat((byte)0xA5, checked((int)extent.Length)).ToArray()); + } + + archive.Position = 0; + var wiped = descriptor.WipeUnusedSpace(archive); + Assert.That(wiped, Is.EqualTo(free.Sum(extent => extent.Length))); + + archive.Position = 0; + Assert.That(descriptor.ExtractEntryToMemory(archive, "DATA.PRG", null), Is.EqualTo(payload)); + var bytes = archive.ToArray(); + foreach (var extent in free) + Assert.That( + bytes.AsSpan(checked((int)extent.Offset), checked((int)extent.Length)).ToArray().All(value => value == 0), + Is.True, + $"Expected free extent '{extent.FileName}' to be zeroed."); + } + private static MemoryStream Create( LynxFormatDescriptor descriptor, IReadOnlyList inputs, From 937f84c60e9fd58325be4bbebefe145bc4b01a07 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:00:10 +0200 Subject: [PATCH 024/165] Fix T64 entry metadata model --- FileFormats/FileFormat.T64/T64Entry.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/FileFormats/FileFormat.T64/T64Entry.cs b/FileFormats/FileFormat.T64/T64Entry.cs index 8de898239..ffca02331 100644 --- a/FileFormats/FileFormat.T64/T64Entry.cs +++ b/FileFormats/FileFormat.T64/T64Entry.cs @@ -2,9 +2,13 @@ namespace FileFormat.T64; /// -/// Represents a t 64 entry. +/// Represents a T64 directory entry. /// public sealed class T64Entry { + /// + /// Gets the zero-based directory slot index. + /// + public int DirectoryIndex { get; init; } /// /// Gets or sets the name. /// @@ -22,6 +26,10 @@ public sealed class T64Entry { /// public byte EntryType { get; init; } // 1=normal, 3=snapshot /// + /// Gets the Commodore file type byte. + /// + public byte FileType { get; init; } + /// /// Gets or sets the start address. /// public ushort StartAddress { get; init; } From a007a54b382fd44e21f2d957cee2df75f40b8c6c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:00:34 +0200 Subject: [PATCH 025/165] Implement managed LZ4 block codec for CSO/ZSO --- FileFormats/FileFormat.Cso/Lz4BlockCodec.cs | 171 ++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 FileFormats/FileFormat.Cso/Lz4BlockCodec.cs diff --git a/FileFormats/FileFormat.Cso/Lz4BlockCodec.cs b/FileFormats/FileFormat.Cso/Lz4BlockCodec.cs new file mode 100644 index 000000000..d984108e6 --- /dev/null +++ b/FileFormats/FileFormat.Cso/Lz4BlockCodec.cs @@ -0,0 +1,171 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; + +namespace FileFormat.Cso; + +/// +/// Minimal raw-LZ4 block codec used by ZSO and CSO v2. +/// +/// +/// Clean-room implementation from the published LZ4 block format. This is deliberately a block +/// codec, not an LZ4 frame codec: CSO/ZSO supply block boundaries and the decoded size themselves. +/// +internal static class Lz4BlockCodec { + private const int HashBits = 16; + private const int HashSize = 1 << HashBits; + private const int MinimumMatch = 4; + private const int MaximumOffset = ushort.MaxValue; + private const int LastLiterals = 5; + private const int MatchFindLimit = 12; + + /// Compresses one independent LZ4 block. + public static byte[] Compress(ReadOnlySpan source) { + if (source.IsEmpty) + return []; + + var table = new int[HashSize]; + Array.Fill(table, -1); + using var output = new MemoryStream(source.Length); + + var anchor = 0; + var current = 0; + var searchLimit = source.Length - MatchFindLimit; + while (current <= searchLimit) { + var hash = Hash(source, current); + var candidate = table[hash]; + table[hash] = current; + + if (candidate < 0 || current - candidate > MaximumOffset || + !source.Slice(candidate, MinimumMatch).SequenceEqual(source.Slice(current, MinimumMatch))) { + ++current; + continue; + } + + // A compliant LZ4 block keeps its final five source bytes literal and starts the final + // match at least twelve bytes before the end. The search limit above enforces the latter; + // this boundary enforces the former. + var matchEnd = current + MinimumMatch; + var candidateEnd = candidate + MinimumMatch; + var maximumMatchEnd = source.Length - LastLiterals; + while (matchEnd < maximumMatchEnd && source[candidateEnd] == source[matchEnd]) { + ++matchEnd; + ++candidateEnd; + } + + var literalLength = current - anchor; + var matchLength = matchEnd - current; + WriteSequence(output, source.Slice(anchor, literalLength), current - candidate, matchLength); + + var previous = current; + current = matchEnd; + anchor = current; + + // Seed hashes inside the match. This is not required for validity, but makes repeated + // structured sectors compress usefully without introducing a heavyweight search chain. + for (var position = previous + 1; position < current && position <= searchLimit; ++position) + table[Hash(source, position)] = position; + } + + WriteLastLiterals(output, source[anchor..]); + return output.ToArray(); + } + + /// Decompresses one raw LZ4 block to exactly bytes. + public static byte[] Decompress(ReadOnlySpan source, int expectedSize) { + if (expectedSize < 0) + throw new ArgumentOutOfRangeException(nameof(expectedSize)); + if (expectedSize == 0) + return []; + + var output = new byte[expectedSize]; + var inputOffset = 0; + var outputOffset = 0; + + while (outputOffset < expectedSize) { + if (inputOffset >= source.Length) + throw new InvalidDataException("Truncated LZ4 block: missing sequence token."); + + var token = source[inputOffset++]; + var literalLength = ReadLength(source, ref inputOffset, token >> 4); + if (literalLength > source.Length - inputOffset || literalLength > expectedSize - outputOffset) + throw new InvalidDataException("Invalid LZ4 block: literal run exceeds its input or output boundary."); + + source.Slice(inputOffset, literalLength).CopyTo(output.AsSpan(outputOffset)); + inputOffset += literalLength; + outputOffset += literalLength; + + // A final sequence consists solely of literals. CSO/ZSO alignment padding may follow it in + // the indexed span, so once the requested decoded block is complete the remaining bytes are + // intentionally ignored. + if (outputOffset == expectedSize) + return output; + + if (source.Length - inputOffset < 2) + throw new InvalidDataException("Truncated LZ4 block: missing match offset."); + var matchOffset = BinaryPrimitives.ReadUInt16LittleEndian(source[inputOffset..]); + inputOffset += 2; + if (matchOffset == 0 || matchOffset > outputOffset) + throw new InvalidDataException("Invalid LZ4 block: match offset points before the decoded prefix."); + + var matchLength = ReadLength(source, ref inputOffset, token & 0x0F) + MinimumMatch; + if (matchLength > expectedSize - outputOffset) + throw new InvalidDataException("Invalid LZ4 block: match run exceeds the decoded block size."); + + var matchSource = outputOffset - matchOffset; + for (var i = 0; i < matchLength; ++i) + output[outputOffset++] = output[matchSource + i]; + } + + return output; + } + + private static void WriteSequence(Stream output, ReadOnlySpan literals, int offset, int matchLength) { + var matchCode = matchLength - MinimumMatch; + var token = (byte)((Math.Min(literals.Length, 15) << 4) | Math.Min(matchCode, 15)); + output.WriteByte(token); + if (literals.Length >= 15) + WriteExtendedLength(output, literals.Length - 15); + output.Write(literals); + + Span offsetBytes = stackalloc byte[2]; + BinaryPrimitives.WriteUInt16LittleEndian(offsetBytes, checked((ushort)offset)); + output.Write(offsetBytes); + if (matchCode >= 15) + WriteExtendedLength(output, matchCode - 15); + } + + private static void WriteLastLiterals(Stream output, ReadOnlySpan literals) { + output.WriteByte((byte)(Math.Min(literals.Length, 15) << 4)); + if (literals.Length >= 15) + WriteExtendedLength(output, literals.Length - 15); + output.Write(literals); + } + + private static void WriteExtendedLength(Stream output, int remaining) { + while (remaining >= 255) { + output.WriteByte(255); + remaining -= 255; + } + output.WriteByte((byte)remaining); + } + + private static int ReadLength(ReadOnlySpan source, ref int offset, int nibble) { + if (nibble < 15) + return nibble; + + var result = 15; + while (true) { + if (offset >= source.Length) + throw new InvalidDataException("Truncated LZ4 block: missing extended length byte."); + var value = source[offset++]; + result = checked(result + value); + if (value != 255) + return result; + } + } + + private static int Hash(ReadOnlySpan source, int offset) { + var value = BinaryPrimitives.ReadUInt32LittleEndian(source.Slice(offset, sizeof(uint))); + return (int)(value * 2654435761u >> (32 - HashBits)); + } +} From cd87e684a65d914932677997ad85ee67c0a76d1e Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:00:45 +0200 Subject: [PATCH 026/165] Harden T64 parsing and recover legacy end addresses --- FileFormats/FileFormat.T64/T64Reader.cs | 135 +++++++++++++++++++----- 1 file changed, 106 insertions(+), 29 deletions(-) diff --git a/FileFormats/FileFormat.T64/T64Reader.cs b/FileFormats/FileFormat.T64/T64Reader.cs index 51372fe51..335697993 100644 --- a/FileFormats/FileFormat.T64/T64Reader.cs +++ b/FileFormats/FileFormat.T64/T64Reader.cs @@ -16,17 +16,39 @@ public sealed class T64Reader : IDisposable { /// public IReadOnlyList Entries => _entries; /// + /// Gets the T64 format version. + /// + public ushort Version { get; private set; } + /// + /// Gets the number of directory slots reserved by the image. + /// + public ushort DirectoryEntryCount { get; private set; } + /// + /// Gets the used-entry count stored in the header. + /// + public ushort UsedEntryCount { get; private set; } + /// /// Gets or sets the tape name. /// public string TapeName { get; private set; } = ""; private const int HeaderSize = 64; private const int EntrySize = 32; + private const ushort Conv64BrokenEndAddress = 0xC3C6; /// /// Initializes a new instance of . /// public T64Reader(Stream stream, bool leaveOpen = false) { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead) + throw new ArgumentException("T64: stream must be readable.", nameof(stream)); + if (stream.CanSeek) { + if (stream.Length > int.MaxValue) + throw new InvalidDataException("T64: images larger than 2 GiB are not supported by the in-memory reader."); + stream.Position = 0; + } + using var ms = new MemoryStream(); stream.CopyTo(ms); _data = ms.ToArray(); @@ -37,67 +59,122 @@ private void Parse() { if (_data.Length < HeaderSize) throw new InvalidDataException("T64: file too small."); - // Validate magic - var sig = Encoding.ASCII.GetString(_data, 0, 3); - if (sig != "C64") + var signature = Encoding.ASCII.GetString(_data, 0, 32).TrimEnd('\0', ' '); + if (!IsKnownSignature(signature)) throw new InvalidDataException("T64: invalid magic."); - var maxEntries = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(34)); - var usedEntries = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(36)); + Version = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(32)); + DirectoryEntryCount = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(34)); + UsedEntryCount = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(36)); + + var directoryEnd = checked(HeaderSize + DirectoryEntryCount * EntrySize); + if (directoryEnd > _data.Length) + throw new InvalidDataException("T64: directory extends beyond end of image."); // Tape name (24 bytes at offset 40) TapeName = Encoding.ASCII.GetString(_data, 40, 24).TrimEnd('\0', ' '); - // Read directory entries - for (var i = 0; i < maxEntries; i++) { + var rawEntries = new List(); + for (var i = 0; i < DirectoryEntryCount; i++) { var off = HeaderSize + i * EntrySize; - if (off + EntrySize > _data.Length) break; - var entryType = _data[off]; - if (entryType == 0) continue; // free slot + if (entryType == 0) continue; + var fileType = _data[off + 1]; var startAddr = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(off + 2)); var endAddr = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(off + 4)); - var dataOffset = (int)BinaryPrimitives.ReadUInt32LittleEndian(_data.AsSpan(off + 8)); + var rawDataOffset = BinaryPrimitives.ReadUInt32LittleEndian(_data.AsSpan(off + 8)); + if (rawDataOffset > int.MaxValue) + throw new InvalidDataException($"T64: entry {i} data offset is outside the supported image range."); + var dataOffset = (int)rawDataOffset; + if (dataOffset < directoryEnd || dataOffset > _data.Length) + throw new InvalidDataException($"T64: entry {i} data offset is outside the payload region."); - // Filename: 16 bytes at offset 16 within entry var nameBytes = _data.AsSpan(off + 16, 16); - var nameEnd = 16; - for (var j = 15; j >= 0; j--) { - if (nameBytes[j] != 0x20 && nameBytes[j] != 0x00) { nameEnd = j + 1; break; } - if (j == 0) nameEnd = 0; - } - var name = Encoding.ASCII.GetString(_data, off + 16, nameEnd); + var nameLength = 16; + while (nameLength > 0 && nameBytes[nameLength - 1] is 0x00 or 0x20) + --nameLength; + var name = Encoding.ASCII.GetString(nameBytes[..nameLength]); + + rawEntries.Add(new RawEntry(i, entryType, fileType, startAddr, endAddr, dataOffset, name)); + } - var size = endAddr > startAddr ? endAddr - startAddr : 0; + var ordered = rawEntries.OrderBy(static e => e.DataOffset).ToArray(); + for (var i = 0; i < ordered.Length; i++) { + var raw = ordered[i]; + var nextOffset = i + 1 < ordered.Length ? ordered[i + 1].DataOffset : _data.Length; + if (nextOffset < raw.DataOffset) + throw new InvalidDataException($"T64: entry {raw.DirectoryIndex} has an invalid payload order."); + + var physicalBytesAvailable = nextOffset - raw.DataOffset; + var size = ResolveSize(raw, physicalBytesAvailable); + if (size < 0 || size > physicalBytesAvailable) + throw new InvalidDataException($"T64: entry {raw.DirectoryIndex} extends into the next payload or past EOF."); _entries.Add(new T64Entry { - Name = name, + DirectoryIndex = raw.DirectoryIndex, + Name = raw.Name, Size = size, - EntryType = entryType, - StartAddress = startAddr, - EndAddress = endAddr, - DataOffset = dataOffset, + EntryType = raw.EntryType, + FileType = raw.FileType, + StartAddress = raw.StartAddress, + EndAddress = raw.EndAddress, + DataOffset = raw.DataOffset, }); } + + _entries.Sort(static (a, b) => a.DirectoryIndex.CompareTo(b.DirectoryIndex)); } + private static long ResolveSize(RawEntry entry, int physicalBytesAvailable) { + // CONV64 historically wrote $C3C6 into every end-address field. VICE and + // t64fix recover such records from the next payload offset (or EOF for the + // final entry), so mirror that public behaviour without depending on their + // implementation code. + if (entry.EntryType == 1 && entry.EndAddress == Conv64BrokenEndAddress) + return physicalBytesAvailable; + + // T64 stores a 16-bit exclusive end address. An entry that ends exactly at + // $10000 therefore wraps the stored field to zero; this is the only valid + // wrap and is needed for payloads reaching the top of C64 memory. + var logicalEnd = entry.EndAddress == 0 && entry.StartAddress != 0 + ? 0x10000L + : entry.EndAddress; + if (logicalEnd < entry.StartAddress) + throw new InvalidDataException($"T64: entry {entry.DirectoryIndex} has an end address before its start address."); + + return logicalEnd - entry.StartAddress; + } + + private static bool IsKnownSignature(string signature) + => signature.StartsWith("C64S tape image file", StringComparison.Ordinal) + || signature.StartsWith("C64 tape image file", StringComparison.Ordinal) + || signature.StartsWith("C64S tape file", StringComparison.Ordinal) + || signature.StartsWith("C64S image file", StringComparison.Ordinal); + /// /// Decodes the supplied input. /// public byte[] Extract(T64Entry entry) { ArgumentNullException.ThrowIfNull(entry); if (entry.Size == 0) return []; + if (entry.DataOffset < 0 || entry.Size > int.MaxValue || (long)entry.DataOffset + entry.Size > _data.Length) + throw new InvalidDataException("T64: entry payload is outside the image."); - var offset = entry.DataOffset; - var length = (int)entry.Size; - if (offset + length > _data.Length) - length = Math.Max(0, _data.Length - offset); - return _data.AsSpan(offset, length).ToArray(); + return _data.AsSpan(entry.DataOffset, (int)entry.Size).ToArray(); } /// /// Releases resources held by this instance. /// public void Dispose() { } + + private readonly record struct RawEntry( + int DirectoryIndex, + byte EntryType, + byte FileType, + ushort StartAddress, + ushort EndAddress, + int DataOffset, + string Name); } From 85eb7693a267cb7d3eb498ea71be6c7648aaad70 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:00:56 +0200 Subject: [PATCH 027/165] ReFS: fix refcount row alignment mask typing --- FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs b/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs index 5dde4eef6..485b70875 100644 --- a/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs +++ b/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs @@ -129,7 +129,7 @@ public static void RefreshTotal(Span value) { } private static void ValidateRangeStart(ulong startVirtualLcn) { - if ((startVirtualLcn & (EntriesPerRow - 1UL)) != 0) + if ((startVirtualLcn & ((ulong)EntriesPerRow - 1UL)) != 0) throw new ArgumentOutOfRangeException( nameof(startVirtualLcn), startVirtualLcn, From 665bdbc7a5de6583384992a1e619864ec6fc070f Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:01:00 +0200 Subject: [PATCH 028/165] Preserve T64 metadata in writer --- FileFormats/FileFormat.T64/T64Writer.cs | 66 +++++++++++++++---------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/FileFormats/FileFormat.T64/T64Writer.cs b/FileFormats/FileFormat.T64/T64Writer.cs index 4fbe970a1..67a7d0f9b 100644 --- a/FileFormats/FileFormat.T64/T64Writer.cs +++ b/FileFormats/FileFormat.T64/T64Writer.cs @@ -8,75 +8,89 @@ namespace FileFormat.T64; /// Writes a Commodore 64 T64 tape container, building the tape record and the directory of program entries. /// public sealed class T64Writer { - private readonly List<(string Name, ushort StartAddress, byte[] Data)> _files = []; + private readonly List<(string Name, ushort StartAddress, byte FileType, byte[] Data)> _files = []; /// /// Performs the add file operation. /// - public void AddFile(string name, ushort startAddress, byte[] data) => - _files.Add((name, startAddress, data)); + public void AddFile(string name, ushort startAddress, byte[] data) + => AddFile(name, startAddress, 0x82, data); + + /// + /// Adds a file while preserving its Commodore file-type byte. + /// + public void AddFile(string name, ushort startAddress, byte fileType, byte[] data) { + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(data); + _files.Add((name, startAddress, fileType, data)); + } /// /// Performs the add file operation. /// - public void AddFile(string name, byte[] data) => - _files.Add((name, 0x0801, data)); // default BASIC start address + public void AddFile(string name, byte[] data) + => AddFile(name, 0x0801, 0x82, data); // default BASIC start address + PRG type /// /// Performs the build operation. /// - public byte[] Build(string tapeName = "TAPE") { - var headerSize = 64; - var dirSize = _files.Count * 32; + public byte[] Build(string tapeName = "TAPE", ushort version = 0x0100) { + ArgumentNullException.ThrowIfNull(tapeName); + if (_files.Count > ushort.MaxValue) + throw new InvalidOperationException("T64: directory entry count exceeds the 16-bit header field."); + + const int headerSize = 64; + const int entrySize = 32; + var dirSize = checked(_files.Count * entrySize); var dataStart = headerSize + dirSize; + // A tape entry addresses its file inside the C64's 64 KB memory map, so no - // entry can be larger than that — and the archive is built in memory. - // Summing the sizes in an int overflowed before either limit was reported. + // entry can be larger than that. The archive itself is currently built in + // memory, therefore it also has to fit a managed byte array. var totalBytes = (long)dataStart; - foreach (var (name, startAddr, data) in _files) { + foreach (var (name, startAddr, _, data) in _files) { if (startAddr + (long)data.Length > 0x10000) throw new InvalidOperationException( $"T64: '{name}' is {data.Length:N0} bytes and loads at ${startAddr:X4}, past the end of " + "the C64's 64 KB address space that a tape entry's start/end addresses describe."); totalBytes += data.Length; + if (totalBytes > Array.MaxLength) + throw new InvalidOperationException("T64: image is too large for the in-memory writer."); } - var totalSize = (int)totalBytes; + var totalSize = checked((int)totalBytes); var output = new byte[totalSize]; // Header var sig = "C64S tape image file\0\0\0\0\0\0\0\0\0\0\0\0"u8; sig[..32].CopyTo(output); - BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(32), 0x0100); // version - BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(34), (ushort)_files.Count); // max entries - BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(36), (ushort)_files.Count); // used entries + BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(32), version); + BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(34), (ushort)_files.Count); + BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(36), (ushort)_files.Count); // Tape name (24 bytes at offset 40) - var nameBytes = Encoding.ASCII.GetBytes(tapeName.Length > 24 ? tapeName[..24] : tapeName); - nameBytes.CopyTo(output, 40); - for (var j = nameBytes.Length; j < 24; j++) - output[40 + j] = 0x20; + var tapeNameBytes = Encoding.ASCII.GetBytes(tapeName.Length > 24 ? tapeName[..24] : tapeName); + tapeNameBytes.CopyTo(output, 40); + output.AsSpan(40 + tapeNameBytes.Length, 24 - tapeNameBytes.Length).Fill(0x20); // Directory entries + data var dataOffset = dataStart; for (var i = 0; i < _files.Count; i++) { - var (name, startAddr, data) = _files[i]; - var entryOff = headerSize + i * 32; + var (name, startAddr, fileType, data) = _files[i]; + var entryOff = headerSize + i * entrySize; output[entryOff] = 1; // normal entry - output[entryOff + 1] = 0x82; // PRG + output[entryOff + 1] = fileType; BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(entryOff + 2), startAddr); - BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(entryOff + 4), (ushort)(startAddr + data.Length)); + BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(entryOff + 4), unchecked((ushort)(startAddr + data.Length))); BinaryPrimitives.WriteUInt32LittleEndian(output.AsSpan(entryOff + 8), (uint)dataOffset); // Filename var fnameBytes = Encoding.ASCII.GetBytes(name.Length > 16 ? name[..16] : name); fnameBytes.CopyTo(output, entryOff + 16); - for (var j = fnameBytes.Length; j < 16; j++) - output[entryOff + 16 + j] = 0x20; + output.AsSpan(entryOff + 16 + fnameBytes.Length, 16 - fnameBytes.Length).Fill(0x20); - // Copy data data.CopyTo(output, dataOffset); dataOffset += data.Length; } From f313d93e3e9f4141b0b8cc4053cd9e658b9d450e Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:01:23 +0200 Subject: [PATCH 029/165] Make T64 block moves overlap-safe --- FileFormats/FileFormat.T64/T64BlockMover.cs | 119 ++++++++++++++------ 1 file changed, 87 insertions(+), 32 deletions(-) diff --git a/FileFormats/FileFormat.T64/T64BlockMover.cs b/FileFormats/FileFormat.T64/T64BlockMover.cs index bb8157cb9..bc80375bf 100644 --- a/FileFormats/FileFormat.T64/T64BlockMover.cs +++ b/FileFormats/FileFormat.T64/T64BlockMover.cs @@ -17,34 +17,44 @@ public sealed class T64BlockMover : IFilesystemBlockMover { /// public void MoveExtent(Stream image, long srcOffset, long dstOffset, long length, bool zeroSource = false) { - if (length <= 0 || srcOffset == dstOffset) return; + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("T64: block moves require a readable, writable, seekable stream.", nameof(image)); + if (srcOffset < 0) throw new ArgumentOutOfRangeException(nameof(srcOffset)); + if (dstOffset < 0) throw new ArgumentOutOfRangeException(nameof(dstOffset)); + if (length < 0) throw new ArgumentOutOfRangeException(nameof(length)); + if (length == 0 || srcOffset == dstOffset) return; + + var srcEnd = checked(srcOffset + length); + var dstEnd = checked(dstOffset + length); + if (srcEnd > image.Length) + throw new ArgumentOutOfRangeException(nameof(length), "T64: source extent extends beyond EOF."); + if (dstEnd > image.Length) + image.SetLength(dstEnd); + var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); try { - var remaining = length; - var src = srcOffset; - var dst = dstOffset; - while (remaining > 0) { - var chunk = (int)Math.Min(remaining, buffer.Length); - image.Position = src; - image.ReadExactly(buffer, 0, chunk); - image.Position = dst; - image.Write(buffer, 0, chunk); - src += chunk; - dst += chunk; - remaining -= chunk; - } - if (zeroSource) { - Array.Clear(buffer, 0, buffer.Length); - remaining = length; - src = srcOffset; - while (remaining > 0) { - var chunk = (int)Math.Min(remaining, buffer.Length); - image.Position = src; - image.Write(buffer, 0, chunk); - src += chunk; - remaining -= chunk; - } + // memmove semantics: a forward-overlapping destination must be copied + // high-to-low or the first write destroys bytes we have not read yet. + if (dstOffset > srcOffset && dstOffset < srcEnd) + CopyBackward(image, srcOffset, dstOffset, length, buffer); + else + CopyForward(image, srcOffset, dstOffset, length, buffer); + + if (!zeroSource) return; + + Array.Clear(buffer, 0, buffer.Length); + var overlapStart = Math.Max(srcOffset, dstOffset); + var overlapEnd = Math.Min(srcEnd, dstEnd); + if (overlapStart >= overlapEnd) { + ZeroRange(image, srcOffset, length, buffer); + return; } + + // Never zero the intersection: those bytes are now part of the moved + // destination. Scrub only the portions exclusively owned by the old run. + ZeroRange(image, srcOffset, overlapStart - srcOffset, buffer); + ZeroRange(image, overlapEnd, srcEnd - overlapEnd, buffer); } finally { ArrayPool.Shared.Return(buffer); } @@ -52,33 +62,78 @@ public void MoveExtent(Stream image, long srcOffset, long dstOffset, long length /// public void UpdateAllocationAfterMove(Stream image, string fileName, long oldOffset, long newOffset, long length) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(fileName); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("T64: allocation updates require a readable, writable, seekable stream.", nameof(image)); + if (oldOffset < 0 || oldOffset > uint.MaxValue) throw new ArgumentOutOfRangeException(nameof(oldOffset)); + if (newOffset < 0 || newOffset > uint.MaxValue) throw new ArgumentOutOfRangeException(nameof(newOffset)); + image.Position = 34; Span hdrBuf = stackalloc byte[2]; image.ReadExactly(hdrBuf); var maxEntries = BinaryPrimitives.ReadUInt16LittleEndian(hdrBuf); + if ((long)HeaderSize + maxEntries * EntrySize > image.Length) + throw new InvalidDataException("T64: directory extends beyond end of image."); + Span entry = stackalloc byte[EntrySize]; for (var i = 0; i < maxEntries; i++) { var slotOff = HeaderSize + i * EntrySize; - var entry = new byte[EntrySize]; image.Position = slotOff; image.ReadExactly(entry); if (entry[0] == 0) continue; - var dataOffset = BinaryPrimitives.ReadUInt32LittleEndian(entry.AsSpan(8)); + var dataOffset = BinaryPrimitives.ReadUInt32LittleEndian(entry[8..]); if (dataOffset != (uint)oldOffset) continue; - // Optionally match by name. if (!string.Equals(fileName, "*", StringComparison.Ordinal)) { - var entryName = Encoding.ASCII.GetString(entry, 16, 16).TrimEnd('\0', ' '); + var entryName = Encoding.ASCII.GetString(entry[16..32]).TrimEnd('\0', ' '); if (!entryName.Equals(fileName, StringComparison.OrdinalIgnoreCase)) continue; } - // Patch data offset. - BinaryPrimitives.WriteUInt32LittleEndian(entry.AsSpan(8), (uint)newOffset); + BinaryPrimitives.WriteUInt32LittleEndian(entry[8..], (uint)newOffset); image.Position = slotOff; image.Write(entry); - break; + return; + } + + throw new InvalidDataException($"T64: no directory entry points at payload offset {oldOffset}."); + } + + private static void CopyForward(Stream image, long src, long dst, long length, byte[] buffer) { + var cursor = 0L; + while (cursor < length) { + var chunk = (int)Math.Min(length - cursor, buffer.Length); + image.Position = src + cursor; + image.ReadExactly(buffer, 0, chunk); + image.Position = dst + cursor; + image.Write(buffer, 0, chunk); + cursor += chunk; + } + } + + private static void CopyBackward(Stream image, long src, long dst, long length, byte[] buffer) { + var remaining = length; + while (remaining > 0) { + var chunk = (int)Math.Min(remaining, buffer.Length); + var offset = remaining - chunk; + image.Position = src + offset; + image.ReadExactly(buffer, 0, chunk); + image.Position = dst + offset; + image.Write(buffer, 0, chunk); + remaining = offset; + } + } + + private static void ZeroRange(Stream image, long offset, long length, byte[] zeroBuffer) { + var remaining = length; + while (remaining > 0) { + var chunk = (int)Math.Min(remaining, zeroBuffer.Length); + image.Position = offset; + image.Write(zeroBuffer, 0, chunk); + offset += chunk; + remaining -= chunk; } } } From 454311dda8222e935e9777cf101d9a93014d355a Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:01:31 +0200 Subject: [PATCH 030/165] Add CSO/ZSO parser and logical block reader --- FileFormats/FileFormat.Cso/CsoImage.cs | 285 +++++++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 FileFormats/FileFormat.Cso/CsoImage.cs diff --git a/FileFormats/FileFormat.Cso/CsoImage.cs b/FileFormats/FileFormat.Cso/CsoImage.cs new file mode 100644 index 000000000..fe27f7344 --- /dev/null +++ b/FileFormats/FileFormat.Cso/CsoImage.cs @@ -0,0 +1,285 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.IO.Compression; +using System.Text; + +namespace FileFormat.Cso; + +internal enum CsoVariant { + CsoV1, + CsoV2, + Zso, +} + +internal enum CsoBlockEncoding { + Stored, + Deflate, + Lz4, +} + +internal sealed record CsoLayout( + long FullSize, + CsoVariant Variant, + string Magic, + uint HeaderSize, + ulong UncompressedSize, + uint BlockSize, + byte Version, + byte Align, + int BlockCount, + uint[] IndexRaw) { + + public long IndexEnd => CsoWriter.HeaderSize + this.IndexRaw.Length * sizeof(uint); + + public long DataStart => this.IndexRaw.Length == 0 + ? this.IndexEnd + : CsoImage.OffsetOf(this.IndexRaw[0], this.Align); + + public long DataEnd => CsoImage.OffsetOf(this.IndexRaw[^1], this.Align); +} + +/// Shared structural parser and logical block reader for CSO v1/v2 and ZSO. +internal static class CsoImage { + internal const int MaximumBlockCount = 8_000_000; + + public static CsoLayout ReadLayout(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead || !stream.CanSeek) + throw new InvalidDataException("CSO/ZSO parsing requires a readable, seekable stream."); + if (stream.Length < CsoWriter.HeaderSize) + throw new InvalidDataException("CSO/ZSO image is shorter than its 24-byte header."); + + stream.Position = 0; + Span header = stackalloc byte[CsoWriter.HeaderSize]; + ReadExact(stream, header); + + var magic = Encoding.ASCII.GetString(header[..4]); + var headerSize = BinaryPrimitives.ReadUInt32LittleEndian(header[4..8]); + var uncompressedSize = BinaryPrimitives.ReadUInt64LittleEndian(header[8..16]); + var blockSize = BinaryPrimitives.ReadUInt32LittleEndian(header[16..20]); + var version = header[20]; + var align = header[21]; + + var variant = (magic, version) switch { + ("CISO", 0 or 1) => CsoVariant.CsoV1, + ("CISO", 2) => CsoVariant.CsoV2, + ("ZISO", 1) => CsoVariant.Zso, + ("CISO", _) => throw new NotSupportedException($"Unsupported CSO version {version}."), + ("ZISO", _) => throw new NotSupportedException($"Unsupported ZSO version {version}; ZSO defines version 1."), + _ => throw new InvalidDataException("Not a CSO/ZSO image (missing CISO/ZISO magic)."), + }; + + if (variant is CsoVariant.CsoV2 or CsoVariant.Zso && headerSize != CsoWriter.HeaderSize) + throw new InvalidDataException($"{magic} v{version} requires header_size=24, got {headerSize}."); + if (blockSize == 0) + throw new InvalidDataException("CSO/ZSO block_size is zero."); + if (blockSize > int.MaxValue) + throw new InvalidDataException($"CSO/ZSO block_size {blockSize} is too large for managed buffers."); + if (align > 31) + throw new InvalidDataException($"CSO/ZSO index_shift {align} is not representable safely."); + + var blockCount64 = uncompressedSize / blockSize + (uncompressedSize % blockSize == 0 ? 0UL : 1UL); + if (blockCount64 > MaximumBlockCount) + throw new InvalidDataException($"CSO/ZSO block count implausible: {blockCount64}."); + var blockCount = checked((int)blockCount64); + var indexCount = checked(blockCount + 1); + var indexBytesLength = checked(indexCount * sizeof(uint)); + if (CsoWriter.HeaderSize + (long)indexBytesLength > stream.Length) + throw new InvalidDataException("CSO/ZSO index table extends past end of file."); + + var indexBytes = new byte[indexBytesLength]; + stream.Position = CsoWriter.HeaderSize; + ReadExact(stream, indexBytes); + var index = new uint[indexCount]; + for (var i = 0; i < index.Length; ++i) + index[i] = BinaryPrimitives.ReadUInt32LittleEndian(indexBytes.AsSpan(i * sizeof(uint), sizeof(uint))); + + if (variant == CsoVariant.CsoV2 && (index[^1] & CsoWriter.IndexUncompressedFlag) != 0) + throw new InvalidDataException("CSO v2 final index entry must not carry the method flag."); + + var minimumDataOffset = CsoWriter.HeaderSize + (long)indexBytesLength; + long previous = -1; + for (var i = 0; i < index.Length; ++i) { + var offset = OffsetOf(index[i], align); + if (offset < minimumDataOffset) + throw new InvalidDataException($"CSO/ZSO index {i} points inside the header/index table."); + if (offset < previous) + throw new InvalidDataException($"CSO/ZSO index is not monotonic at entry {i}."); + if (offset > stream.Length) + throw new InvalidDataException($"CSO/ZSO index {i} points past end of file."); + previous = offset; + } + + return new CsoLayout( + stream.Length, variant, magic, headerSize, uncompressedSize, blockSize, version, align, + blockCount, index); + } + + public static (long Offset, long Length, CsoBlockEncoding Encoding) GetBlockSpan(CsoLayout layout, int blockIndex) { + ArgumentNullException.ThrowIfNull(layout); + if ((uint)blockIndex >= (uint)layout.BlockCount) + throw new ArgumentOutOfRangeException(nameof(blockIndex)); + + var raw = layout.IndexRaw[blockIndex]; + var offset = OffsetOf(raw, layout.Align); + var next = OffsetOf(layout.IndexRaw[blockIndex + 1], layout.Align); + var length = next - offset; + if (length < 0 || next > layout.FullSize) + throw new InvalidDataException($"CSO/ZSO block {blockIndex} has an invalid physical span."); + + var encoding = layout.Variant switch { + CsoVariant.CsoV1 => (raw & CsoWriter.IndexUncompressedFlag) != 0 + ? CsoBlockEncoding.Stored : CsoBlockEncoding.Deflate, + CsoVariant.Zso => (raw & CsoWriter.IndexUncompressedFlag) != 0 + ? CsoBlockEncoding.Stored : CsoBlockEncoding.Lz4, + CsoVariant.CsoV2 => length >= layout.BlockSize + ? CsoBlockEncoding.Stored + : (raw & CsoWriter.IndexUncompressedFlag) != 0 ? CsoBlockEncoding.Lz4 : CsoBlockEncoding.Deflate, + _ => throw new InvalidDataException("Unknown CSO/ZSO variant."), + }; + return (offset, length, encoding); + } + + public static int LogicalBlockLength(CsoLayout layout, int blockIndex) { + if ((uint)blockIndex >= (uint)layout.BlockCount) + throw new ArgumentOutOfRangeException(nameof(blockIndex)); + var start = checked((ulong)blockIndex * layout.BlockSize); + var remaining = layout.UncompressedSize - start; + return checked((int)Math.Min(remaining, layout.BlockSize)); + } + + /// + /// Decodes one block to its full block_size slab. The final logical block may use only a prefix; + /// writers conventionally pad the remainder before compression and consumers truncate by + /// uncompressed_size. + /// + public static byte[] DecodeBlock(Stream stream, CsoLayout layout, int blockIndex) { + var (offset, length, encoding) = GetBlockSpan(layout, blockIndex); + if (length > int.MaxValue) + throw new InvalidDataException($"CSO/ZSO block {blockIndex} is too large to decode."); + var encoded = new byte[(int)length]; + stream.Position = offset; + ReadExact(stream, encoded); + + var blockSize = checked((int)layout.BlockSize); + switch (encoding) { + case CsoBlockEncoding.Stored: { + var logicalLength = LogicalBlockLength(layout, blockIndex); + if (encoded.Length < logicalLength) + throw new InvalidDataException($"Stored CSO/ZSO block {blockIndex} is shorter than its logical bytes."); + var result = new byte[blockSize]; + encoded.AsSpan(0, Math.Min(encoded.Length, blockSize)).CopyTo(result); + return result; + } + case CsoBlockEncoding.Deflate: + return InflateExact(encoded, blockSize, blockIndex); + case CsoBlockEncoding.Lz4: + return Lz4BlockCodec.Decompress(encoded, blockSize); + default: + throw new InvalidDataException("Unknown CSO/ZSO block encoding."); + } + } + + public static Stream OpenLogicalStream(Stream image, CsoLayout layout, IReadOnlyDictionary? replacements = null) + => new LogicalStream(image, layout, replacements); + + internal static long OffsetOf(uint rawIndex, byte align) { + var units = rawIndex & CsoWriter.IndexOffsetMask; + var offset = (ulong)units << align; + if (offset > long.MaxValue) + throw new InvalidDataException("CSO/ZSO index offset exceeds the supported stream range."); + return (long)offset; + } + + private static byte[] InflateExact(ReadOnlySpan source, int expectedSize, int blockIndex) { + using var input = new MemoryStream(source.ToArray(), writable: false); + using var deflate = new DeflateStream(input, CompressionMode.Decompress, leaveOpen: false); + var output = new byte[expectedSize]; + var written = 0; + while (written < output.Length) { + var read = deflate.Read(output, written, output.Length - written); + if (read <= 0) + break; + written += read; + } + if (written != expectedSize) + throw new InvalidDataException( + $"DEFLATE block {blockIndex} decoded to {written} bytes, expected block_size {expectedSize}."); + return output; + } + + internal static void ReadExact(Stream stream, Span destination) { + var read = 0; + while (read < destination.Length) { + var count = stream.Read(destination[read..]); + if (count <= 0) + throw new EndOfStreamException("Unexpected end of CSO/ZSO stream."); + read += count; + } + } + + private sealed class LogicalStream( + Stream image, + CsoLayout layout, + IReadOnlyDictionary? replacements) : Stream { + private int _blockIndex; + private byte[]? _block; + private int _offsetInBlock; + private long _position; + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => checked((long)layout.UncompressedSize); + public override long Position { + get => this._position; + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + => this.Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) { + if (buffer.IsEmpty || this._position >= this.Length) + return 0; + + var total = 0; + while (!buffer.IsEmpty && this._position < this.Length) { + if (this._block == null) { + if (this._blockIndex >= layout.BlockCount) + throw new InvalidDataException("CSO/ZSO logical stream ended before uncompressed_size."); + if (replacements != null && replacements.TryGetValue(this._blockIndex, out var replacement)) { + if (replacement.Length != layout.BlockSize) + throw new InvalidDataException( + $"Replacement block {this._blockIndex} has {replacement.Length} bytes; expected {layout.BlockSize}."); + this._block = replacement; + } else { + this._block = DecodeBlock(image, layout, this._blockIndex); + } + this._offsetInBlock = 0; + } + + var logicalLength = LogicalBlockLength(layout, this._blockIndex); + var available = logicalLength - this._offsetInBlock; + var take = Math.Min(buffer.Length, available); + this._block.AsSpan(this._offsetInBlock, take).CopyTo(buffer); + buffer = buffer[take..]; + this._offsetInBlock += take; + this._position += take; + total += take; + + if (this._offsetInBlock == logicalLength) { + this._block = null; + this._offsetInBlock = 0; + ++this._blockIndex; + } + } + return total; + } + + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } +} From 0ce5a9240ec2ed93656d4767ea0f7c01766cfeef Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:01:32 +0200 Subject: [PATCH 031/165] Unify T64 mutation paths --- FileFormats/FileFormat.T64/T64Modifier.cs | 123 ++-------------------- 1 file changed, 7 insertions(+), 116 deletions(-) diff --git a/FileFormats/FileFormat.T64/T64Modifier.cs b/FileFormats/FileFormat.T64/T64Modifier.cs index b99041e76..fb3216418 100644 --- a/FileFormats/FileFormat.T64/T64Modifier.cs +++ b/FileFormats/FileFormat.T64/T64Modifier.cs @@ -1,129 +1,20 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; -using System.Text; namespace FileFormat.T64; /// -/// In-place T64 modifier — performs O(touched bytes) random-access I/O against a -/// T64 tape image. T64 has a 64-byte header followed by a fixed-size directory -/// table of N×32-byte slots, then concatenated file data. -/// -/// AddFile: finds an empty slot (entryType=0) in the directory, -/// appends file data at EOF, and fills in the slot. -/// RemoveFile: sets the slot's entryType to 0 (marks it free). -/// Data is left in place (no compaction). +/// Compatibility facade for the T64 in-place editor. /// public static class T64Modifier { - - private const int HeaderSize = 64; - private const int EntrySize = 32; - /// - /// Adds a file to an existing T64 tape image. Finds the first free slot - /// (entryType=0) in the directory, appends the file data at the end of the - /// image, and writes the directory entry. + /// Adds or replaces a file in an existing T64 tape image. /// - public static void AddFile(Stream image, string name, byte[] data, ushort startAddress = 0x0801) { - ArgumentNullException.ThrowIfNull(image); - ArgumentNullException.ThrowIfNull(name); - ArgumentNullException.ThrowIfNull(data); - - // Read header to get maxEntries. - image.Position = 34; - Span hdrBuf = stackalloc byte[4]; - image.ReadExactly(hdrBuf); - var maxEntries = BinaryPrimitives.ReadUInt16LittleEndian(hdrBuf); - var usedEntries = BinaryPrimitives.ReadUInt16LittleEndian(hdrBuf.Slice(2)); - - // Scan the directory for a free slot. - var freeSlot = -1; - for (var i = 0; i < maxEntries; i++) { - var slotOff = HeaderSize + i * EntrySize; - image.Position = slotOff; - var typeByte = image.ReadByte(); - if (typeByte <= 0) { // 0 or EOF - freeSlot = i; - break; - } - } - - if (freeSlot < 0) - throw new IOException($"T64: no free directory slot (max {maxEntries})."); - - // Append data at EOF. - var dataOffset = (int)image.Length; - image.Position = dataOffset; - image.Write(data); - - // Build directory entry (32 bytes). - var entry = new byte[EntrySize]; - entry[0] = 1; // entryType = normal - entry[1] = 0x82; // C64 file type = PRG - - BinaryPrimitives.WriteUInt16LittleEndian(entry.AsSpan(2), startAddress); - BinaryPrimitives.WriteUInt16LittleEndian(entry.AsSpan(4), (ushort)(startAddress + data.Length)); - BinaryPrimitives.WriteUInt32LittleEndian(entry.AsSpan(8), (uint)dataOffset); - - // Filename: 16 bytes, space-padded. - var nameBytes = Encoding.ASCII.GetBytes(name.Length > 16 ? name[..16] : name); - nameBytes.CopyTo(entry, 16); - for (var j = nameBytes.Length; j < 16; j++) - entry[16 + j] = 0x20; - - // Write the directory entry. - image.Position = HeaderSize + freeSlot * EntrySize; - image.Write(entry); - - // Update usedEntries count. - usedEntries++; - image.Position = 36; - Span usedBuf = stackalloc byte[2]; - BinaryPrimitives.WriteUInt16LittleEndian(usedBuf, usedEntries); - image.Write(usedBuf); - } + public static void AddFile(Stream image, string name, byte[] data, ushort startAddress = 0x0801) + => T64InPlaceModifier.AddFile(image, name, data, startAddress); /// - /// Removes a named file from the T64 image by zeroing its directory entry type. - /// Returns false if not found. + /// Removes a named file from the T64 image and compacts the vacated bytes. /// - public static bool RemoveFile(Stream image, string name) { - ArgumentNullException.ThrowIfNull(image); - ArgumentNullException.ThrowIfNull(name); - - image.Position = 34; - Span hdrBuf = stackalloc byte[4]; - image.ReadExactly(hdrBuf); - var maxEntries = BinaryPrimitives.ReadUInt16LittleEndian(hdrBuf); - var usedEntries = BinaryPrimitives.ReadUInt16LittleEndian(hdrBuf.Slice(2)); - - for (var i = 0; i < maxEntries; i++) { - var slotOff = HeaderSize + i * EntrySize; - var entry = new byte[EntrySize]; - image.Position = slotOff; - image.ReadExactly(entry); - - if (entry[0] == 0) continue; // free slot - - // Read filename (16 bytes at offset 16). - var entryName = Encoding.ASCII.GetString(entry, 16, 16).TrimEnd('\0', ' '); - if (!entryName.Equals(name, StringComparison.OrdinalIgnoreCase)) continue; - - // Mark slot as free. - entry[0] = 0; - image.Position = slotOff; - image.Write(entry); - - // Decrement used count. - if (usedEntries > 0) usedEntries--; - image.Position = 36; - Span usedBuf = stackalloc byte[2]; - BinaryPrimitives.WriteUInt16LittleEndian(usedBuf, usedEntries); - image.Write(usedBuf); - - return true; - } - - return false; - } + public static bool RemoveFile(Stream image, string name) + => T64InPlaceModifier.RemoveFile(image, name); } From bd5d6d89089c62f74daf304c8f7d8874293c0919 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:01:37 +0200 Subject: [PATCH 032/165] + harden TI-TXT parsing and preserve sparse sections --- .../FileFormat.FirmwareHex/TiTxtReader.cs | 78 ++++++++++++++----- 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/FileFormats/FileFormat.FirmwareHex/TiTxtReader.cs b/FileFormats/FileFormat.FirmwareHex/TiTxtReader.cs index f7b676ce3..fe4e3b1a1 100644 --- a/FileFormats/FileFormat.FirmwareHex/TiTxtReader.cs +++ b/FileFormats/FileFormat.FirmwareHex/TiTxtReader.cs @@ -4,53 +4,93 @@ namespace FileFormat.FirmwareHex; /// -/// Reader for TI-TXT MSP430 text firmware files. Addresses are introduced by -/// @HHHH lines; data lines follow as space-separated hex bytes (typically -/// 16 per line); a single q token terminates the file. +/// Parser for Texas Instruments TI-TXT firmware images. Address records introduce +/// sparse byte runs and a line containing only q terminates the document. /// public sealed class TiTxtReader { - /// Parses a TI-TXT document into a . + /// Parses a TI-TXT document into its sparse firmware image. public static FirmwareImage Read(string text) { - var segments = new SortedDictionary>(); - uint? currentAddr = null; + ArgumentNullException.ThrowIfNull(text); + + var sections = new List<(uint Address, List Data)>(); + List? currentData = null; var sawTerminator = false; var dataLines = 0; using var reader = new StringReader(text); - for (var line = reader.ReadLine(); line != null; line = reader.ReadLine()) { - line = line.Trim(); + for (var raw = reader.ReadLine(); raw != null; raw = reader.ReadLine()) { + var line = raw.Trim(); if (line.Length == 0) continue; - if (line.StartsWith('q') || line.StartsWith('Q')) { + if (sawTerminator) + throw new InvalidDataException($"TiTxt: content after the 'q' termination line ('{line}')."); + + if (line.Equals("q", StringComparison.OrdinalIgnoreCase)) { sawTerminator = true; - break; + currentData = null; + continue; } + if (line.StartsWith('@')) { - if (!uint.TryParse(line.AsSpan(1).Trim(), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var addr)) + var address = line.AsSpan(1).Trim(); + if (address.IsEmpty || !uint.TryParse(address, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var parsed)) throw new InvalidDataException($"TiTxt: invalid address record '{line}'."); - currentAddr = addr; - if (!segments.ContainsKey(addr)) segments[addr] = []; + currentData = []; + sections.Add((parsed, currentData)); continue; } - if (currentAddr is null) + + if (currentData is null) throw new InvalidDataException($"TiTxt: data before first '@address' line ('{line}')."); dataLines++; foreach (var token in line.Split([' ', '\t'], StringSplitOptions.RemoveEmptyEntries)) { - if (!byte.TryParse(token, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var b)) + if (token.Length != 2 || !byte.TryParse(token, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var value)) throw new InvalidDataException($"TiTxt: invalid hex byte '{token}'."); - segments[currentAddr.Value].Add(b); + currentData.Add(value); } } if (!sawTerminator) throw new InvalidDataException("TiTxt: missing 'q' termination line."); - var dict = new SortedDictionary(); - foreach (var (k, v) in segments) dict[k] = v.ToArray(); - var (merged, gapCount, totalBytes) = IntelHexReader.MergeSegments(dict); + var merged = NormalizeSections(sections, out var gapCount, out var totalBytes); return new FirmwareImage(merged, StartAddress: null, RecordCount: dataLines, GapCount: gapCount, TotalDataBytes: totalBytes, SourceFormat: "TiTxt"); } + + private static List<(uint Address, byte[] Data)> NormalizeSections( + List<(uint Address, List Data)> sections, out int gapCount, out int totalBytes) { + var result = new List<(uint Address, byte[] Data)>(); + gapCount = 0; + totalBytes = 0; + ulong previousEnd = 0; + var havePrevious = false; + + foreach (var (address, bytes) in sections.Where(section => section.Data.Count != 0).OrderBy(section => section.Address)) { + var end = (ulong)address + (ulong)bytes.Count; + if (end > (ulong)uint.MaxValue + 1) + throw new InvalidDataException($"TiTxt: section at 0x{address:X} extends beyond the 32-bit address space."); + if (havePrevious && address < previousEnd) + throw new InvalidDataException($"TiTxt: section at 0x{address:X} overlaps a preceding section ending at 0x{previousEnd - 1:X}."); + + if (havePrevious && address == previousEnd) { + var previous = result[^1]; + var joined = new byte[previous.Data.Length + bytes.Count]; + previous.Data.CopyTo(joined, 0); + bytes.CopyTo(joined, previous.Data.Length); + result[^1] = (previous.Address, joined); + } else { + if (havePrevious) gapCount++; + result.Add((address, bytes.ToArray())); + } + + previousEnd = end; + havePrevious = true; + checked { totalBytes += bytes.Count; } + } + + return result; + } } From 28d82474e6c0b8430099b3c1e4862e59e2726ce6 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:01:42 +0200 Subject: [PATCH 033/165] Prevent NRG sector edits from overwriting trailer --- .../FileFormat.Nrg/NrgInPlaceModifier.cs | 356 +++++++----------- 1 file changed, 135 insertions(+), 221 deletions(-) diff --git a/FileFormats/FileFormat.Nrg/NrgInPlaceModifier.cs b/FileFormats/FileFormat.Nrg/NrgInPlaceModifier.cs index f3f53ec7b..948c549b0 100644 --- a/FileFormats/FileFormat.Nrg/NrgInPlaceModifier.cs +++ b/FileFormats/FileFormat.Nrg/NrgInPlaceModifier.cs @@ -1,333 +1,247 @@ #pragma warning disable CS1591 +using System.Buffers.Binary; using System.Globalization; namespace FileFormat.Nrg; /// -/// In-place sector-rewrite modifier for a Nero Burning ROM NRG disc image. -/// Operates at the raw 2 048-byte user-data region of each CD sector at the -/// fixed byte offset lba * sectorSize + dataOffset, where -/// sectorSize and dataOffset are the geometry detected from the -/// data area (raw 2 352 Mode 1, raw 2 352 Mode 2 Form 1, 2 336-byte sectors, -/// or flat 2 048-byte cooked sectors). -/// -/// NRG framing. An NRG image is a stream of CD sectors followed -/// by a footer at EOF identifying the format version: -/// -/// NRG v2: last 12 bytes — "NER5" + uint64 BE chunk-table offset. -/// NRG v1: last 8 bytes — "NERO" + uint32 BE chunk-table offset. -/// -/// The footer is preserved byte-identical across in-place rewrites and is -/// relocated past the new EOF whenever the data area grows. -/// -/// Scope. Rewrites only the user-data bytes inside an existing -/// sector or appends a brand-new sector at the end of the data area. It does -/// not understand the inner ISO 9660 directory structure — that is the -/// job of / its reader. Synthetic entry -/// names of the form sector-NNNNNN.bin address a single sector LBA. -/// Multi-track DAOI/CUEX layouts are not parsed — the modifier treats the -/// stream as a single track of sectors at flat LBA offsets. Sync pattern -/// (12 B), 3-byte address, 1-byte mode, and the EDC/ECC tail of raw sectors -/// are preserved on rewrite and synthesised on append. -/// -/// True in-place. Writes touch only the 2 048-byte user-data -/// region of the targeted sector. Bytes outside that region — header bytes of -/// the same sector, every untouched sector, the system area (LBA 0-15), the -/// PVD at LBA 16, the ISO root directory, and the trailing NRG footer — stay -/// byte-identical at their original byte offsets (the footer migrates to -/// follow the new EOF when the data area grows). +/// Low-level fixed-LBA sector editor for single-track NRG images whose ISO data +/// track begins at file offset zero. This is intentionally narrower than the +/// descriptor's public file-level R/W path: ordinary file CRUD uses a verified +/// extract/edit/re-create rebuild so ISO directory metadata stays coherent. /// public static class NrgInPlaceModifier { - - /// How this format names itself when it has to refuse something. private const string Label = "NRG"; - - private const int Iso9660SectorSize = 2048; private const int RawSectorSize = 2352; private const int SectorSize2336 = 2336; private const int PvdLba = 16; private const int Mode1DataOffset = 16; private const int Mode2Form1DataOffset = 24; - private const int Ner5FooterSize = 12; private const int NeroFooterSize = 8; - private static readonly byte[] MagicNer5 = [(byte)'N', (byte)'E', (byte)'R', (byte)'5']; - private static readonly byte[] MagicNero = [(byte)'N', (byte)'E', (byte)'R', (byte)'O']; - - /// The 12-byte CD-ROM sync pattern (00 FF*10 00) prefixed to every - /// raw 2 352-byte sector. - private static readonly byte[] Sync = [ + private static ReadOnlySpan Sync => [ 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, ]; /// - /// Detected on-disk sector geometry for an NRG image. - /// is the byte offset within a sector where the 2 048 B of ISO user data - /// begins. excludes the trailing NRG footer - /// (12 bytes for v2, 8 bytes for v1) when present. + /// Detected sector geometry. is the byte offset of + /// the 2,048-byte user-data region. is the byte + /// offset at which the NRG descriptor begins, not merely EOF minus the footer. /// public readonly record struct SectorGeometry(int SectorSize, int DataOffset, long DataAreaLength); + private readonly record struct Trailer(long Offset, bool HasNrgFooter); + /// - /// Detects the sector geometry of the same way - /// does — by probing for the CD001 PVD - /// signature at LBA 16 inside the data area. Falls back to raw Mode 1 - /// (2 352 / 16) when no probe succeeds. NRG v2 ("NER5") and v1 ("NERO") - /// footers are excluded from the data area. + /// Detects the single-track geometry from the ISO PVD at LBA 16. The NERO/ + /// NER5 footer's trailer pointer is honoured, so chunk metadata is never + /// mistaken for sector data. /// public static SectorGeometry DetectGeometry(Stream image) { ArgumentNullException.ThrowIfNull(image); - var dataLen = DetectDataAreaLength(image); - if (TryProbe(image, RawSectorSize, Mode1DataOffset, dataLen)) return new(RawSectorSize, Mode1DataOffset, dataLen); - if (TryProbe(image, RawSectorSize, Mode2Form1DataOffset, dataLen)) return new(RawSectorSize, Mode2Form1DataOffset, dataLen); - if (TryProbe(image, SectorSize2336, 8, dataLen)) return new(SectorSize2336, 8, dataLen); - if (TryProbe(image, Iso9660SectorSize, 0, dataLen)) return new(Iso9660SectorSize, 0, dataLen); - return new(RawSectorSize, Mode1DataOffset, dataLen); + if (!image.CanRead || !image.CanSeek) + throw new ArgumentException("NRG sector editing requires a readable, seekable stream.", nameof(image)); + + var trailer = ReadTrailer(image); + var dataLength = trailer.Offset; + if (TryProbe(image, RawSectorSize, Mode1DataOffset, dataLength)) + return new(RawSectorSize, Mode1DataOffset, dataLength); + if (TryProbe(image, RawSectorSize, Mode2Form1DataOffset, dataLength)) + return new(RawSectorSize, Mode2Form1DataOffset, dataLength); + if (TryProbe(image, SectorSize2336, 8, dataLength)) + return new(SectorSize2336, 8, dataLength); + if (TryProbe(image, Iso9660SectorSize, 0, dataLength)) + return new(Iso9660SectorSize, 0, dataLength); + + throw new NotSupportedException( + $"{Label}: the fixed-LBA editor supports only a single ISO data track beginning at file offset zero."); } - private static long DetectDataAreaLength(Stream image) { - // Probe NER5 first (longer footer). + private static Trailer ReadTrailer(Stream image) { if (image.Length >= Ner5FooterSize) { image.Position = image.Length - Ner5FooterSize; - Span tail = stackalloc byte[Ner5FooterSize]; - if (image.Read(tail) == Ner5FooterSize && - tail[0] == MagicNer5[0] && tail[1] == MagicNer5[1] && - tail[2] == MagicNer5[2] && tail[3] == MagicNer5[3]) - return image.Length - Ner5FooterSize; + Span footer = stackalloc byte[Ner5FooterSize]; + if (ReadExactly(image, footer) && footer[..4].SequenceEqual("NER5"u8)) { + var offset = BinaryPrimitives.ReadUInt64BigEndian(footer[4..]); + if (offset <= (ulong)(image.Length - Ner5FooterSize)) + return new(checked((long)offset), true); + } } + if (image.Length >= NeroFooterSize) { image.Position = image.Length - NeroFooterSize; - Span tail = stackalloc byte[NeroFooterSize]; - if (image.Read(tail) == NeroFooterSize && - tail[0] == MagicNero[0] && tail[1] == MagicNero[1] && - tail[2] == MagicNero[2] && tail[3] == MagicNero[3]) - return image.Length - NeroFooterSize; + Span footer = stackalloc byte[NeroFooterSize]; + if (ReadExactly(image, footer) && footer[..4].SequenceEqual("NERO"u8)) { + var offset = BinaryPrimitives.ReadUInt32BigEndian(footer[4..]); + if (offset <= image.Length - NeroFooterSize) + return new(offset, true); + } } - return image.Length; + + return new(image.Length, false); } private static bool TryProbe(Stream image, int sectorSize, int dataOffset, long dataAreaLength) { var pvdAt = (long)PvdLba * sectorSize + dataOffset; - if (pvdAt + 6 > dataAreaLength) return false; + if (pvdAt > dataAreaLength - 6) + return false; + image.Position = pvdAt; - Span sig = stackalloc byte[6]; - var read = image.Read(sig); - if (read < 6) return false; - return sig[0] == 1 && sig[1] == (byte)'C' && sig[2] == (byte)'D' && - sig[3] == (byte)'0' && sig[4] == (byte)'0' && sig[5] == (byte)'1'; + Span signature = stackalloc byte[6]; + return ReadExactly(image, signature) && + signature[0] == 1 && signature[1..].SequenceEqual("CD001"u8); } /// - /// Rewrites the 2 048-byte user-data region of sector - /// in place. Other bytes — sync/header/EDC for raw sectors, every other - /// sector, every other region of the image, and the trailing NRG footer — - /// are untouched. If points past the current - /// data-area EOF, the image is grown sector-by-sector with appended-sector - /// framing () and the footer is relocated to the - /// new EOF. + /// Rewrites one existing 2,048-byte user-data sector. If the requested LBA + /// would extend a real NRG image, the operation is refused because moving the + /// descriptor requires rewriting ETN/DAO offsets and sizes as well. /// - /// When .Length - /// differs from 2 048. public static void WriteSector(Stream image, int lba, ReadOnlySpan userData) { - ArgumentNullException.ThrowIfNull(image); - if (lba < 0) throw new ArgumentOutOfRangeException(nameof(lba)); - if (userData.Length != Iso9660SectorSize) - throw new ArgumentException( - $"Sector user data must be exactly {Iso9660SectorSize} bytes; got {userData.Length}.", - nameof(userData)); - var geom = DetectGeometry(image); - WriteSector(image, lba, userData, geom); + var geometry = DetectGeometry(image); + WriteSector(image, lba, userData, geometry); } - /// - /// Variant of that - /// reuses a previously-probed geometry, avoiding a redundant PVD probe per - /// call when a caller is rewriting several sectors back-to-back. - /// - public static void WriteSector(Stream image, int lba, ReadOnlySpan userData, SectorGeometry geom) { + /// Rewrites a sector using an already detected geometry. + public static void WriteSector(Stream image, int lba, ReadOnlySpan userData, SectorGeometry geometry) { ArgumentNullException.ThrowIfNull(image); - if (lba < 0) throw new ArgumentOutOfRangeException(nameof(lba)); + if (!image.CanWrite || !image.CanSeek) + throw new ArgumentException("NRG sector editing requires a writable, seekable stream.", nameof(image)); + if (lba < 0) + throw new ArgumentOutOfRangeException(nameof(lba)); if (userData.Length != Iso9660SectorSize) - throw new ArgumentException( - $"Sector user data must be exactly {Iso9660SectorSize} bytes; got {userData.Length}.", - nameof(userData)); + throw new ArgumentException($"Sector user data must be exactly {Iso9660SectorSize} bytes; got {userData.Length}.", nameof(userData)); - var sectorStart = (long)lba * geom.SectorSize; - var endOfSector = sectorStart + geom.SectorSize; - - if (endOfSector <= geom.DataAreaLength) { - image.Position = sectorStart + geom.DataOffset; + var sectorStart = checked((long)lba * geometry.SectorSize); + var sectorEnd = checked(sectorStart + geometry.SectorSize); + if (sectorEnd <= geometry.DataAreaLength) { + image.Position = checked(sectorStart + geometry.DataOffset); image.Write(userData); return; } - AppendSector(image, lba, userData, geom); + AppendSector(image, lba, userData, geometry); } /// - /// Extends the data area so that sector exists, - /// writing as its 2 048-byte payload. - /// Intermediate sectors are appended with format-correct framing. The - /// trailing NRG footer is preserved verbatim and rewritten at the new EOF. + /// Appends sectors only to a footer-less raw image. Extending an NRG image is + /// not a byte-local operation because its descriptor records track offsets and + /// sizes; callers wanting growth must use the descriptor's rebuild editor. /// - public static void AppendSector(Stream image, int lba, ReadOnlySpan userData, SectorGeometry geom) { + public static void AppendSector(Stream image, int lba, ReadOnlySpan userData, SectorGeometry geometry) { ArgumentNullException.ThrowIfNull(image); - if (lba < 0) throw new ArgumentOutOfRangeException(nameof(lba)); + if (lba < 0) + throw new ArgumentOutOfRangeException(nameof(lba)); if (userData.Length != Iso9660SectorSize) - throw new ArgumentException( - $"Sector user data must be exactly {Iso9660SectorSize} bytes; got {userData.Length}.", - nameof(userData)); - - byte[]? footer = null; - if (geom.DataAreaLength < image.Length) { - footer = new byte[image.Length - geom.DataAreaLength]; - image.Position = geom.DataAreaLength; - var got = 0; - while (got < footer.Length) { - var r = image.Read(footer, got, footer.Length - got); - if (r == 0) break; - got += r; - } - } + throw new ArgumentException($"Sector user data must be exactly {Iso9660SectorSize} bytes; got {userData.Length}.", nameof(userData)); - var firstMissingLba = (int)((geom.DataAreaLength + geom.SectorSize - 1) / geom.SectorSize); - var lastLbaToWrite = lba; + if (ReadTrailer(image).HasNrgFooter) + throw new NotSupportedException( + $"{Label}: extending a chunked image requires relocating and rewriting its track descriptor; use file-level rebuild modification instead."); - for (var i = firstMissingLba; i < lastLbaToWrite; i++) { - image.Position = (long)i * geom.SectorSize; - WriteFramedSector(image, geom, ReadOnlySpan.Empty); + var firstMissingLba = checked((int)((geometry.DataAreaLength + geometry.SectorSize - 1) / geometry.SectorSize)); + for (var current = firstMissingLba; current < lba; ++current) { + image.Position = checked((long)current * geometry.SectorSize); + WriteFramedSector(image, geometry, ReadOnlySpan.Empty); } - image.Position = (long)lba * geom.SectorSize; - WriteFramedSector(image, geom, userData); - - if (footer != null) { - var newDataEnd = (long)(lba + 1) * geom.SectorSize; - image.SetLength(newDataEnd + footer.Length); - image.Position = newDataEnd; - image.Write(footer); - } else { - image.SetLength((long)(lba + 1) * geom.SectorSize); - } + image.Position = checked((long)lba * geometry.SectorSize); + WriteFramedSector(image, geometry, userData); + image.SetLength(checked((long)(lba + 1) * geometry.SectorSize)); } - private static void WriteFramedSector(Stream image, SectorGeometry geom, ReadOnlySpan userData) { - var sector = new byte[geom.SectorSize]; - switch (geom.SectorSize) { - case RawSectorSize: - Sync.AsSpan().CopyTo(sector.AsSpan(0, 12)); - sector[15] = (byte)(geom.DataOffset == Mode2Form1DataOffset ? 0x02 : 0x01); - break; - case SectorSize2336: - break; + private static void WriteFramedSector(Stream image, SectorGeometry geometry, ReadOnlySpan userData) { + var sector = new byte[geometry.SectorSize]; + if (geometry.SectorSize == RawSectorSize) { + Sync.CopyTo(sector); + sector[15] = geometry.DataOffset == Mode2Form1DataOffset ? (byte)0x02 : (byte)0x01; } if (!userData.IsEmpty) - userData.CopyTo(sector.AsSpan(geom.DataOffset, Iso9660SectorSize)); + userData.CopyTo(sector.AsSpan(geometry.DataOffset, Iso9660SectorSize)); image.Write(sector); } - /// - /// Zeros the 2 048-byte user-data region of sector - /// in place. The sector framing bytes and the trailing NRG footer are - /// preserved; only the user data is wiped. Returns true if the - /// sector existed (and was zeroed), false if - /// is past the data-area EOF. - /// + /// Zeros one existing sector's 2,048-byte user-data region. public static bool ZeroSector(Stream image, int lba) { - ArgumentNullException.ThrowIfNull(image); - if (lba < 0) return false; - var geom = DetectGeometry(image); - return ZeroSector(image, lba, geom); + if (lba < 0) + return false; + var geometry = DetectGeometry(image); + return ZeroSector(image, lba, geometry); } - /// - /// Variant of reusing a previously-probed geometry. - /// - public static bool ZeroSector(Stream image, int lba, SectorGeometry geom) { + /// Zeros one existing sector using an already detected geometry. + public static bool ZeroSector(Stream image, int lba, SectorGeometry geometry) { ArgumentNullException.ThrowIfNull(image); - if (lba < 0) return false; - var endOfSector = (long)lba * geom.SectorSize + geom.SectorSize; - if (endOfSector > geom.DataAreaLength) return false; - var zeros = new byte[Iso9660SectorSize]; - image.Position = (long)lba * geom.SectorSize + geom.DataOffset; + if (lba < 0) + return false; + var sectorEnd = checked((long)lba * geometry.SectorSize + geometry.SectorSize); + if (sectorEnd > geometry.DataAreaLength) + return false; + + Span zeros = stackalloc byte[Iso9660SectorSize]; + image.Position = checked((long)lba * geometry.SectorSize + geometry.DataOffset); image.Write(zeros); return true; } - /// - /// Parses a synthetic sector-NNNNNN.bin entry name and returns the - /// embedded sector LBA. Names that don't match the schema return - /// false. - /// + /// Parses a synthetic sector-NNNNNN.bin LBA name. public static bool TryParseSectorEntryName(string entryName, out int lba) { lba = -1; - if (string.IsNullOrEmpty(entryName)) return false; + if (string.IsNullOrEmpty(entryName)) + return false; + var leaf = Path.GetFileName(entryName); const string prefix = "sector-"; const string suffix = ".bin"; - if (!leaf.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; - if (!leaf.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) return false; + if (!leaf.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) || + !leaf.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) + return false; + var numeric = leaf.AsSpan(prefix.Length, leaf.Length - prefix.Length - suffix.Length); return int.TryParse(numeric, NumberStyles.Integer, CultureInfo.InvariantCulture, out lba) && lba >= 0; } - /// - /// Formats a sector LBA into the synthetic entry name used by the - /// in-place modifier. - /// + /// Formats an LBA using the synthetic low-level sector namespace. public static string FormatSectorEntryName(int lba) => string.Create(CultureInfo.InvariantCulture, $"sector-{lba:D6}.bin"); - /// - /// Routes each input through the sector-rewrite path. Inputs whose - /// ArchiveName matches sector-NNNNNN.bin are written at the - /// fixed LBA byte offset. Inputs whose ArchiveName doesn't match the - /// schema are refused — inner ISO 9660 directory mutation is - /// delegated to FileSystem.Iso. - /// + /// Rewrites sectors named through the synthetic low-level namespace. public static void AddOrReplaceSectors(Stream image, IEnumerable<(string ArchiveName, byte[] Data)> inputs) { ArgumentNullException.ThrowIfNull(image); ArgumentNullException.ThrowIfNull(inputs); - var geom = DetectGeometry(image); + var geometry = DetectGeometry(image); foreach (var (name, data) in inputs) { - // A name this cannot place is not a name to pass over. These images list - // the ISO 9660 files inside them, so a caller has every reason to hand one - // back -- and skipping it wrote nothing, raised nothing, and reported the - // add as done. Six files added, six files gone, no error anywhere. if (!TryParseSectorEntryName(name, out var lba)) throw new NotSupportedException( - $"{Label}: '{name}' cannot be added. This image is edited a sector at a time, so an " - + "entry has to be named 'sector-NNNN.bin' for the sector it replaces. Adding a file to " - + "the ISO 9660 filesystem inside the image is not something this supports."); - if (data.Length != Iso9660SectorSize) - throw new ArgumentException( - $"Sector entry '{name}' must carry exactly {Iso9660SectorSize} bytes; got {data.Length}.", - nameof(inputs)); - WriteSector(image, lba, data, geom); - geom = DetectGeometry(image); + $"{Label}: '{name}' is not a fixed-LBA sector name. Use the descriptor's file-level modification path for ISO entries."); + WriteSector(image, lba, data, geometry); } } - /// - /// Zeros each named sector-NNNNNN.bin. Names that don't match the - /// schema are refused; sectors past the data-area EOF are still - /// skipped. The framing bytes of an existing sector and the trailing NRG - /// footer are preserved. - /// + /// Zeros sectors named through the synthetic low-level namespace. public static void RemoveSectors(Stream image, IEnumerable entryNames) { ArgumentNullException.ThrowIfNull(image); ArgumentNullException.ThrowIfNull(entryNames); - var geom = DetectGeometry(image); + var geometry = DetectGeometry(image); foreach (var name in entryNames) { if (!TryParseSectorEntryName(name, out var lba)) throw new NotSupportedException( - $"{Label}: '{name}' cannot be removed. This image is edited a sector at a time, so an " - + "entry has to be named 'sector-NNNN.bin' for the sector it clears. Removing a file from " - + "the ISO 9660 filesystem inside the image is not something this supports."); - ZeroSector(image, lba, geom); + $"{Label}: '{name}' is not a fixed-LBA sector name. Use the descriptor's file-level modification path for ISO entries."); + _ = ZeroSector(image, lba, geometry); } } + + private static bool ReadExactly(Stream stream, Span buffer) { + var offset = 0; + while (offset < buffer.Length) { + var read = stream.Read(buffer[offset..]); + if (read == 0) + return false; + offset += read; + } + return true; + } } From 6d56d0bb2a3c6af09a21310c843d544e792dead8 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:01:58 +0200 Subject: [PATCH 034/165] Harden Intel HEX record validation expression --- FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs b/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs index 590ec2ab6..51aabb216 100644 --- a/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs +++ b/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs @@ -104,7 +104,7 @@ private static void ValidateRecordShape(byte type, ushort address, int payloadLe throw new InvalidDataException( $"IntelHex: record type 0x{type:X2} requires {expectedPayloadLength} data bytes, got {payloadLength} in '{line}'."); - if (type is 0x02 or 0x03 or 0x04 or 0x05 && address != 0) + if ((type is 0x02 or 0x03 or 0x04 or 0x05) && address != 0) throw new InvalidDataException( $"IntelHex: record type 0x{type:X2} requires address 0000, got {address:X4} in '{line}'."); } From b68205fce8997331e8b40bcb42bbab9654becac5 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:02:04 +0200 Subject: [PATCH 035/165] Support CSO v2 and ZSO writing --- FileFormats/FileFormat.Cso/CsoWriter.cs | 237 ++++++++++++++---------- 1 file changed, 138 insertions(+), 99 deletions(-) diff --git a/FileFormats/FileFormat.Cso/CsoWriter.cs b/FileFormats/FileFormat.Cso/CsoWriter.cs index 7d500f43d..8c7731c05 100644 --- a/FileFormats/FileFormat.Cso/CsoWriter.cs +++ b/FileFormats/FileFormat.Cso/CsoWriter.cs @@ -5,139 +5,178 @@ namespace FileFormat.Cso; /// -/// Writes a PSP CSO v1 ("CISO") compressed-ISO container from scratch (WORM). -/// -/// Layout (24-byte header + (N+1)·uint32 index + N compressed blocks): -/// -/// $00..$03: magic "CISO" -/// $04..$07: uint32 LE header_size = 24 -/// $08..$0F: uint64 LE uncompressed_size -/// $10..$13: uint32 LE block_size (this writer uses 2048 = ISO 9660 sector) -/// $14: uint8 version = 1 -/// $15: uint8 align (left-shift applied to index offsets; this writer uses 0) -/// $16..$17: uint16 reserved = 0 -/// $18..(header+4·(N+1)): index table. bit 31 set = stored uncompressed. -/// Each block: raw DEFLATE bytes (no zlib header) of one block_size-aligned slab, -/// OR the slab verbatim when the compressed output is not smaller than the slab. -/// -/// -/// -/// ZSO (LZ4) and CSO v2 are out of scope; see . +/// Writes PSP CSO v1/v2 and ZSO compressed-ISO containers. /// +/// +/// CSO v1 uses raw DEFLATE and bit 31 as the stored/uncompressed flag. ZSO keeps the v1 index +/// semantics but uses raw LZ4 blocks. CSO v2 infers stored blocks from an indexed span greater +/// than or equal to block_size and uses bit 31 to select LZ4 (set) or DEFLATE (clear) for shorter +/// compressed spans. +/// public sealed class CsoWriter { - - /// Default block size — 2048 = one ISO 9660 cooked sector. + /// Default block size — one cooked ISO 9660 sector. public const int DefaultBlockSize = 2048; - /// CSO v1 header length in bytes. + /// CSO/ZSO header length in bytes. internal const int HeaderSize = 24; - /// Bit 31 of an index entry: when set, the block is stored uncompressed. + /// Bit 31 of an index entry: stored for v1/ZSO, LZ4-method for compressed CSO v2. internal const uint IndexUncompressedFlag = 0x8000_0000u; - /// Mask for the offset portion of an index entry. + /// Mask for the shifted file-offset portion of an index entry. internal const uint IndexOffsetMask = 0x7FFF_FFFFu; + /// Builds a canonical CSO v1 stream around . + public static byte[] Build(ReadOnlySpan uncompressedData, int blockSize = DefaultBlockSize) + => Build(uncompressedData, blockSize, CsoVariant.CsoV1); + + internal static byte[] Build(ReadOnlySpan uncompressedData, int blockSize, CsoVariant variant) { + using var input = new MemoryStream(uncompressedData.ToArray(), writable: false); + using var output = new MemoryStream(); + Write(output, input, checked((ulong)uncompressedData.Length), blockSize, variant); + return output.ToArray(); + } + /// - /// Builds a CSO v1 stream that, when fully decompressed, yields - /// . Blocks of - /// bytes are each DEFLATE-compressed; if the compressed output is not smaller - /// than the original slab, the slab is stored verbatim and its index entry - /// gets the bit set. + /// Re-encodes into a canonical align=0 CSO/ZSO stream + /// without buffering the whole ISO in memory. /// - public static byte[] Build(ReadOnlySpan uncompressedData, int blockSize = DefaultBlockSize) { + internal static void Write( + Stream output, Stream uncompressedInput, ulong uncompressedSize, int blockSize, CsoVariant variant) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(uncompressedInput); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("CSO/ZSO output must be writable and seekable.", nameof(output)); + if (!uncompressedInput.CanRead) + throw new ArgumentException("CSO/ZSO input must be readable.", nameof(uncompressedInput)); if (blockSize <= 0) throw new ArgumentOutOfRangeException(nameof(blockSize), "block_size must be positive."); - var uncompressedSize = uncompressedData.Length; - var blockCount = (uncompressedSize + blockSize - 1) / blockSize; - var indexCount = blockCount + 1; // trailing sentinel = EOF offset. + var blockSize64 = checked((ulong)blockSize); + var blockCount64 = uncompressedSize / blockSize64 + (uncompressedSize % blockSize64 == 0 ? 0UL : 1UL); + if (blockCount64 > CsoImage.MaximumBlockCount) + throw new InvalidOperationException($"CSO/ZSO block count is too large: {blockCount64}."); + var blockCount = checked((int)blockCount64); + var index = new uint[checked(blockCount + 1)]; + var dataStart = checked(HeaderSize + index.Length * sizeof(uint)); - var output = new MemoryStream(); - // Reserve space for header + index; we'll patch the index after writing blocks. - output.Write(new byte[HeaderSize + indexCount * 4]); + output.Position = 0; + output.SetLength(dataStart); + output.Position = dataStart; - var index = new uint[indexCount]; + var slab = new byte[blockSize]; + ulong consumed = 0; for (var i = 0; i < blockCount; ++i) { - var rawOffset = i * blockSize; - var rawLen = Math.Min(blockSize, uncompressedSize - rawOffset); - - // If the last block is partial, pad it to blockSize with zero bytes so the - // decoder yields blockSize bytes — its caller truncates by uncompressed_size. - var slab = new byte[blockSize]; - uncompressedData.Slice(rawOffset, rawLen).CopyTo(slab); - - var blockOffset = checked((uint)output.Position); - if (blockOffset > IndexOffsetMask) - throw new InvalidOperationException("CSO output exceeds 2 GiB — would overflow a 31-bit offset."); - index[i] = blockOffset; - - var deflated = Deflate(slab); - if (deflated.Length < blockSize) { - output.Write(deflated, 0, deflated.Length); - } else { - // Compressed payload is not smaller than the slab — store verbatim and - // flag the index entry so the reader knows not to inflate. - output.Write(slab, 0, blockSize); - index[i] |= IndexUncompressedFlag; - } + Array.Clear(slab); + var logicalLength = checked((int)Math.Min(blockSize64, uncompressedSize - consumed)); + ReadExact(uncompressedInput, slab.AsSpan(0, logicalLength)); + consumed += checked((uint)logicalLength); + + var offset = CheckedIndexOffset(output.Position); + var (payload, flag) = EncodeBlock(slab, variant); + index[i] = offset | flag; + output.Write(payload); } - // Sentinel index entry = end-of-stream offset (used by the reader to - // compute the last real block's compressed length via nextOffset-offset). - var eofOffset = checked((uint)output.Position); - if (eofOffset > IndexOffsetMask) - throw new InvalidOperationException("CSO output exceeds 2 GiB — would overflow a 31-bit offset."); - index[blockCount] = eofOffset; + if (consumed != uncompressedSize) + throw new InvalidDataException($"CSO/ZSO encoder consumed {consumed} bytes; expected {uncompressedSize}."); - // Now patch the header + index in place. - var bytes = output.ToArray(); - WriteHeader(bytes, uncompressedSize, blockSize); - for (var i = 0; i < indexCount; ++i) - BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(HeaderSize + i * 4, 4), index[i]); + index[blockCount] = CheckedIndexOffset(output.Position); - return bytes; + Span header = stackalloc byte[HeaderSize]; + WriteHeader(header, uncompressedSize, blockSize, variant); + output.Position = 0; + output.Write(header); + Span entry = stackalloc byte[sizeof(uint)]; + foreach (var value in index) { + BinaryPrimitives.WriteUInt32LittleEndian(entry, value); + output.Write(entry); + } + output.Position = output.Length; } - /// - /// Writes the fixed-layout CSO v1 header into the first 24 bytes of - /// . - /// - internal static void WriteHeader(Span target, long uncompressedSize, int blockSize) { - target[0] = (byte)'C'; - target[1] = (byte)'I'; - target[2] = (byte)'S'; - target[3] = (byte)'O'; - BinaryPrimitives.WriteUInt32LittleEndian(target.Slice(4, 4), HeaderSize); - BinaryPrimitives.WriteUInt64LittleEndian(target.Slice(8, 8), (ulong)uncompressedSize); - BinaryPrimitives.WriteUInt32LittleEndian(target.Slice(16, 4), (uint)blockSize); - target[20] = 1; // version - target[21] = 0; // align - target[22] = 0; // reserved - target[23] = 0; // reserved + internal static void WriteHeader(Span target, ulong uncompressedSize, int blockSize, CsoVariant variant) { + if (target.Length < HeaderSize) + throw new ArgumentException("CSO/ZSO header target is too small.", nameof(target)); + target[..HeaderSize].Clear(); + + var magic = variant == CsoVariant.Zso ? "ZISO"u8 : "CISO"u8; + magic.CopyTo(target); + BinaryPrimitives.WriteUInt32LittleEndian(target[4..8], HeaderSize); + BinaryPrimitives.WriteUInt64LittleEndian(target[8..16], uncompressedSize); + BinaryPrimitives.WriteUInt32LittleEndian(target[16..20], checked((uint)blockSize)); + target[20] = variant == CsoVariant.CsoV2 ? (byte)2 : (byte)1; + target[21] = 0; // canonical writer emits byte-granular indexes (no padding/alignment shift). } - /// Raw-DEFLATE encode (no zlib header). + /// Raw-DEFLATE encode (no zlib wrapper). internal static byte[] Deflate(ReadOnlySpan data) { - using var ms = new MemoryStream(); - using (var ds = new DeflateStream(ms, CompressionLevel.Optimal, leaveOpen: true)) - ds.Write(data); - return ms.ToArray(); + using var output = new MemoryStream(); + using (var deflate = new DeflateStream(output, CompressionLevel.Optimal, leaveOpen: true)) + deflate.Write(data); + return output.ToArray(); } - /// Raw-DEFLATE decode back to its original bytes. + /// Raw-DEFLATE decode helper retained for callers that need a bounded decoded slab. internal static byte[] Inflate(ReadOnlySpan data, int expectedSize) { - using var input = new MemoryStream(data.ToArray()); - using var ds = new DeflateStream(input, CompressionMode.Decompress); + using var input = new MemoryStream(data.ToArray(), writable: false); + using var deflate = new DeflateStream(input, CompressionMode.Decompress); var output = new byte[expectedSize]; var written = 0; - while (written < expectedSize) { - var n = ds.Read(output, written, expectedSize - written); - if (n <= 0) break; - written += n; + while (written < output.Length) { + var read = deflate.Read(output, written, output.Length - written); + if (read <= 0) + break; + written += read; } - if (written < expectedSize) + if (written != output.Length) Array.Resize(ref output, written); return output; } + + private static (byte[] Payload, uint Flag) EncodeBlock(ReadOnlySpan slab, CsoVariant variant) { + switch (variant) { + case CsoVariant.CsoV1: { + var compressed = Deflate(slab); + return compressed.Length < slab.Length + ? (compressed, 0u) + : (slab.ToArray(), IndexUncompressedFlag); + } + case CsoVariant.Zso: { + var compressed = Lz4BlockCodec.Compress(slab); + return compressed.Length < slab.Length + ? (compressed, 0u) + : (slab.ToArray(), IndexUncompressedFlag); + } + case CsoVariant.CsoV2: { + var deflate = Deflate(slab); + var lz4 = Lz4BlockCodec.Compress(slab); + if (lz4.Length < deflate.Length && lz4.Length < slab.Length) + return (lz4, IndexUncompressedFlag); + if (deflate.Length < slab.Length) + return (deflate, 0u); + // CSO v2 does not flag stored blocks: span >= block_size is the stored discriminator. + return (slab.ToArray(), 0u); + } + default: + throw new ArgumentOutOfRangeException(nameof(variant)); + } + } + + private static uint CheckedIndexOffset(long position) { + if (position < 0 || position > IndexOffsetMask) + throw new InvalidOperationException( + "CSO/ZSO output exceeds the 31-bit byte-offset range of an align=0 index."); + return checked((uint)position); + } + + private static void ReadExact(Stream input, Span destination) { + var read = 0; + while (read < destination.Length) { + var count = input.Read(destination[read..]); + if (count <= 0) + throw new EndOfStreamException("Uncompressed CSO/ZSO input ended before uncompressed_size."); + read += count; + } + } } From 4df16fa151164271a2cb6794e50bc1cf412847a0 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:02:10 +0200 Subject: [PATCH 036/165] Avoid 32-bit overflow when flattening firmware images --- FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs b/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs index 23b02dcd3..4717c712d 100644 --- a/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs +++ b/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs @@ -33,11 +33,16 @@ string SourceFormat public byte[] ToFlatBinary(byte fill = 0xFF) { if (this.Segments.Count == 0) return []; var lo = this.Segments.Min(s => s.Address); - var hi = this.Segments.Max(s => s.Address + (uint)s.Data.Length); - var buf = new byte[hi - lo]; + var hi = this.Segments.Max(s => (ulong)s.Address + (uint)s.Data.Length); + var length = hi - lo; + if (length > int.MaxValue) + throw new InvalidDataException( + $"Firmware image spans {length} bytes and cannot be represented as one managed flat byte array."); + + var buf = new byte[(int)length]; if (fill != 0) Array.Fill(buf, fill); foreach (var (addr, data) in this.Segments) - Array.Copy(data, 0, buf, (int)(addr - lo), data.Length); + Array.Copy(data, 0, buf, checked((int)((ulong)addr - lo)), data.Length); return buf; } From adf5654d1dd91fbae584a43be64427b59edefa05 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:02:22 +0200 Subject: [PATCH 037/165] Make CSO/ZSO block edits transactional --- .../FileFormat.Cso/CsoInPlaceModifier.cs | 272 +++--------------- 1 file changed, 36 insertions(+), 236 deletions(-) diff --git a/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs b/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs index ad3a6103b..d860ab836 100644 --- a/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs +++ b/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs @@ -1,263 +1,63 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; namespace FileFormat.Cso; /// -/// In-place block-level mutator for PSP CSO v1 images. Lets callers replace -/// individual decompressed blocks without rewriting the whole container. -/// -/// Semantics for : -/// -/// If the new compressed payload fits inside the old block's on-disk -/// slot, the payload is written at the same offset and trailing slack -/// is zero-padded — the index table and every other block's bytes are -/// unchanged. -/// Otherwise the payload is appended at the current end of stream and -/// the block's index entry is updated to point at the new location. -/// The old in-place bytes become orphaned (defrag-recoverable). -/// -/// -/// -/// CSO v2 / ZSO (LZ4) are out of scope; only the v1 header layout with -/// align=0 is supported. The modifier refuses to operate on streams whose -/// header reports a different version or a non-zero align (because the -/// offset-shift semantics would silently misplace the new block). +/// Block-level mutator for CSO v1/v2 and ZSO images. /// +/// +/// A CSO index is ordered by physical block position and each block's physical size is derived from +/// the following index entry. Growing a middle block in place therefore cannot be represented by +/// changing only that entry. Edits are staged as a canonical whole-container repack and committed +/// only after the replacement has been encoded successfully. This also removes stale padding and +/// supports non-zero index shifts without trying to preserve their allocation slack. +/// public static class CsoInPlaceModifier { - /// - /// Replaces block 's content with - /// (which must be exactly - /// block_size bytes long, matching the container's geometry). The - /// payload is DEFLATE-compressed; if the result is smaller than the slab - /// it's written compressed, otherwise stored uncompressed (with the index - /// entry's high bit set). + /// Replaces one logical block. must be exactly block_size + /// bytes; for a partial final block only its logical prefix is retained, matching CSO semantics. /// - /// Seekable read/write stream over the CSO image. - /// 0-based block index. Must be in - /// [0, block_count). - /// Exactly block_size bytes. public static void WriteBlock(Stream image, int blockIndex, ReadOnlySpan newUncompressedData) { ArgumentNullException.ThrowIfNull(image); - if (!image.CanSeek) - throw new InvalidOperationException("CSO in-place modify requires a seekable stream."); - if (!image.CanWrite) - throw new InvalidOperationException("CSO in-place modify requires a writable stream."); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new InvalidOperationException("CSO/ZSO block modification requires a readable, writable, seekable stream."); - var header = ReadHeader(image); - if (blockIndex < 0 || blockIndex >= header.BlockCount) + var layout = CsoImage.ReadLayout(image); + if ((uint)blockIndex >= (uint)layout.BlockCount) throw new ArgumentOutOfRangeException(nameof(blockIndex), - $"CSO block index {blockIndex} outside [0, {header.BlockCount})."); - if (newUncompressedData.Length != header.BlockSize) + $"CSO/ZSO block index {blockIndex} outside [0, {layout.BlockCount})."); + if (newUncompressedData.Length != layout.BlockSize) throw new ArgumentException( - $"New block payload must be exactly block_size ({header.BlockSize}) bytes; got {newUncompressedData.Length}.", + $"New block payload must be exactly block_size ({layout.BlockSize}) bytes; got {newUncompressedData.Length}.", nameof(newUncompressedData)); - // Compress (or accept verbatim) the new payload. - var deflated = CsoWriter.Deflate(newUncompressedData); - byte[] payload; - bool storedUncompressed; - if (deflated.Length < header.BlockSize) { - payload = deflated; - storedUncompressed = false; - } else { - payload = newUncompressedData.ToArray(); - storedUncompressed = true; - } - - // Determine old block's on-disk extent. - var oldRaw = header.IndexRaw[blockIndex]; - var oldNextRaw = header.IndexRaw[blockIndex + 1]; - var oldOffset = (long)(oldRaw & CsoWriter.IndexOffsetMask) << header.Align; - var oldEndOffset = (long)(oldNextRaw & CsoWriter.IndexOffsetMask) << header.Align; - var oldSize = oldEndOffset - oldOffset; - - if (payload.Length <= oldSize) { - // In-place: write payload at oldOffset, zero trailing slack. - image.Position = oldOffset; - image.Write(payload, 0, payload.Length); - var slack = (int)(oldSize - payload.Length); - if (slack > 0) - image.Write(new byte[slack], 0, slack); - - // Update flag bit on the index entry (length unchanged → next offset - // is also unchanged, so we only need to rewrite this one entry). - var newRaw = (uint)(oldRaw & CsoWriter.IndexOffsetMask); - if (storedUncompressed) - newRaw |= CsoWriter.IndexUncompressedFlag; - WriteIndexEntry(image, blockIndex, newRaw); - } else { - // Append-at-EOF: orphan the old in-place bytes, write payload at EOF, - // patch this block's index entry to point at the new offset. - var appendOffset = image.Length; - if (appendOffset > CsoWriter.IndexOffsetMask) - throw new InvalidOperationException( - "CSO image would exceed 2 GiB after append — would overflow a 31-bit offset."); - image.Position = appendOffset; - image.Write(payload, 0, payload.Length); - - // The sentinel entry (index[blockCount]) is the on-disk EOF marker; it - // changes by the new payload length minus the existing zero-length sliver - // that lived between the old EOF and the previous sentinel. Recompute - // from the post-write stream length to stay consistent. - var newSentinelRaw = (uint)image.Length; - // The block boundary we're moving is the chosen block's start; its - // length is now (newSentinelRaw - appendOffset). For the WriteBlock - // contract to remain valid for the FOLLOWING entries' offsets we must - // also update every index slot that previously pointed past oldOffset - // but lived inside the file — which is none, because we appended. - - var newRaw = (uint)appendOffset; - if (storedUncompressed) - newRaw |= CsoWriter.IndexUncompressedFlag; - WriteIndexEntry(image, blockIndex, newRaw); - - // Update the sentinel (index[blockCount]) to the new EOF so the reader - // computes our block's compressed length as (sentinel - newOffset). - // BUT — the sentinel only acts as the "next offset" for the LAST block. - // For non-last blocks, the next offset is index[blockIndex+1], which - // didn't move (still points at the OLD orphaned-region neighbour). To - // preserve the (next - this) length contract for our moved block, we - // need the block immediately after ours, in index-order, to "see" us - // at the new offset and yield the right length. - // - // Easiest correct approach: when we append, also update index[blockIndex+1] - // to point at the new EOF so reader's (next - this) = (newEOF - appendOffset) - // = payload.Length. But that breaks the neighbour's own length. - // - // Real CSO writers handle this by simply keeping orphans and laying each - // block's index entry pointing to its physical start; the "length" of - // a block is determined either by (next - this) OR by reading until the - // expected slab boundary. The reader in CsoFormatDescriptor uses - // (next - this), so when we append we need to ensure the next entry is - // the sentinel. - // - // Strategy: if blockIndex is NOT the last block, append a copy of the - // following block's CURRENT compressed bytes after our new payload so - // that index[blockIndex+1] = appendOffset + payload.Length still points - // at a valid (now duplicated) copy of that next block. Repeat through - // the end of the index. - RebuildTailFromBlock(image, header, blockIndex, appendOffset, payload.Length); - } - } - - /// - /// After an append-grow on , copies every - /// block past it to the new tail so that the (next - this) length contract - /// holds for the entire index. The trailing sentinel is updated to the - /// final EOF. - /// - private static void RebuildTailFromBlock( - Stream image, CsoHeader oldHeader, int movedBlockIndex, - long movedBlockNewOffset, int movedBlockNewSize) { - - var cursor = movedBlockNewOffset + movedBlockNewSize; - var newIndex = new uint[oldHeader.IndexRaw.Length]; - Array.Copy(oldHeader.IndexRaw, newIndex, oldHeader.IndexRaw.Length); - // The moved block's entry was already rewritten by the caller; preserve it. - newIndex[movedBlockIndex] = (uint)movedBlockNewOffset - | (oldHeader.IndexRaw[movedBlockIndex] & CsoWriter.IndexUncompressedFlag); - - for (var i = movedBlockIndex + 1; i < oldHeader.BlockCount; ++i) { - // Copy block i's existing compressed bytes from its OLD offset to the new tail. - var oldOff = (long)(oldHeader.IndexRaw[i] & CsoWriter.IndexOffsetMask) << oldHeader.Align; - var oldEnd = (long)(oldHeader.IndexRaw[i + 1] & CsoWriter.IndexOffsetMask) << oldHeader.Align; - var oldLen = (int)(oldEnd - oldOff); - if (oldLen < 0) oldLen = 0; - var buf = new byte[oldLen]; - if (oldLen > 0) { - image.Position = oldOff; - ReadExact(image, buf); - } - image.Position = cursor; - image.Write(buf, 0, oldLen); + var replacement = newUncompressedData.ToArray(); + var replacements = new Dictionary { [blockIndex] = replacement }; + using var logical = CsoImage.OpenLogicalStream(image, layout, replacements); + using var staged = CreateScratchStream(); + CsoWriter.Write(staged, logical, layout.UncompressedSize, checked((int)layout.BlockSize), layout.Variant); - if (cursor > CsoWriter.IndexOffsetMask) - throw new InvalidOperationException( - "CSO image would exceed 2 GiB after append — would overflow a 31-bit offset."); - - newIndex[i] = (uint)cursor | (oldHeader.IndexRaw[i] & CsoWriter.IndexUncompressedFlag); - cursor += oldLen; - } - // Sentinel = final EOF. - if (cursor > CsoWriter.IndexOffsetMask) - throw new InvalidOperationException( - "CSO image would exceed 2 GiB after append — would overflow a 31-bit offset."); - newIndex[oldHeader.BlockCount] = (uint)cursor; - image.SetLength(cursor); - - // Persist the rewritten index. - image.Position = CsoWriter.HeaderSize; - Span idxBuf = stackalloc byte[4]; - for (var i = 0; i < newIndex.Length; ++i) { - BinaryPrimitives.WriteUInt32LittleEndian(idxBuf, newIndex[i]); - image.Write(idxBuf); - } + staged.Position = 0; + image.Position = 0; + image.SetLength(0); + staged.CopyTo(image); + image.Flush(); } - // ── Header parsing ──────────────────────────────────────────────────── + // ── Header parsing retained for callers that only need geometry ─────── internal sealed record CsoHeader( uint HeaderSize, ulong UncompressedSize, uint BlockSize, byte Version, byte Align, - int BlockCount, uint[] IndexRaw); + int BlockCount, uint[] IndexRaw, CsoVariant Variant); internal static CsoHeader ReadHeader(Stream image) { - image.Position = 0; - Span hdr = stackalloc byte[CsoWriter.HeaderSize]; - ReadExact(image, hdr); - if (hdr[0] != 'C' || hdr[1] != 'I' || hdr[2] != 'S' || hdr[3] != 'O') - throw new InvalidDataException("Not a CSO v1 image — missing 'CISO' magic."); - var headerSize = BinaryPrimitives.ReadUInt32LittleEndian(hdr.Slice(4, 4)); - var uncompressedSize = BinaryPrimitives.ReadUInt64LittleEndian(hdr.Slice(8, 8)); - var blockSize = BinaryPrimitives.ReadUInt32LittleEndian(hdr.Slice(16, 4)); - var version = hdr[20]; - var align = hdr[21]; - - if (version != 1) - throw new NotSupportedException( - $"CSO in-place modify supports v1 only (got v{version}). CSO v2 / ZSO are deferred."); - if (align != 0) - throw new NotSupportedException( - $"CSO in-place modify requires align=0 (got align={align}). Non-zero align is deferred " + - "because offset shifts would silently misplace a moved block."); - if (blockSize == 0) - throw new InvalidDataException("CSO block_size is zero."); - - var blockCount = (int)((uncompressedSize + blockSize - 1) / blockSize); - var indexCount = blockCount + 1; - var idxBytes = new byte[indexCount * 4]; - image.Position = CsoWriter.HeaderSize; - ReadExact(image, idxBytes); - var indexRaw = new uint[indexCount]; - for (var i = 0; i < indexCount; ++i) - indexRaw[i] = BinaryPrimitives.ReadUInt32LittleEndian(idxBytes.AsSpan(i * 4, 4)); - - return new CsoHeader(headerSize, uncompressedSize, blockSize, version, align, blockCount, indexRaw); - } - - private static void WriteIndexEntry(Stream image, int blockIndex, uint rawEntry) { - image.Position = CsoWriter.HeaderSize + blockIndex * 4L; - Span buf = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32LittleEndian(buf, rawEntry); - image.Write(buf); + var layout = CsoImage.ReadLayout(image); + return new CsoHeader( + layout.HeaderSize, layout.UncompressedSize, layout.BlockSize, layout.Version, layout.Align, + layout.BlockCount, layout.IndexRaw, layout.Variant); } - private static void ReadExact(Stream s, Span dst) { - var read = 0; - while (read < dst.Length) { - var n = s.Read(dst[read..]); - if (n <= 0) throw new EndOfStreamException("Unexpected end of CSO stream."); - read += n; - } - } - - private static void ReadExact(Stream s, byte[] dst) { - var read = 0; - while (read < dst.Length) { - var n = s.Read(dst, read, dst.Length - read); - if (n <= 0) throw new EndOfStreamException("Unexpected end of CSO stream."); - read += n; - } - } + internal static FileStream CreateScratchStream() + => new(Path.Combine(Path.GetTempPath(), "cwb_cso_" + Guid.NewGuid().ToString("N") + ".tmp"), + FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 64 * 1024, FileOptions.DeleteOnClose); } From 2c1eed78ff99f98591d8e0f2ce0de1bb56659c62 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:02:23 +0200 Subject: [PATCH 038/165] Harden T64 in-place mutation --- .../FileFormat.T64/T64InPlaceModifier.cs | 265 +++++++++--------- 1 file changed, 128 insertions(+), 137 deletions(-) diff --git a/FileFormats/FileFormat.T64/T64InPlaceModifier.cs b/FileFormats/FileFormat.T64/T64InPlaceModifier.cs index 5af77eaa2..266978a74 100644 --- a/FileFormats/FileFormat.T64/T64InPlaceModifier.cs +++ b/FileFormats/FileFormat.T64/T64InPlaceModifier.cs @@ -15,29 +15,23 @@ namespace FileFormat.T64; /// /// 0..63 - 64-byte header (signature, version, max-entries, used-entries, tape name). /// 64..64+maxEntries*32 - directory: N * 32-byte slot records (entry type, C64 type, start/end addr, absolute data offset, filename). -/// 64+maxEntries*32 .. EOF - concatenated file payloads addressed by each slot's absolute dataOffset field. +/// 64+maxEntries*32 .. EOF - file payloads addressed by each slot's absolute dataOffset field. /// /// Add: if a directory slot is currently free (entryType=0) the /// new entry fills that slot and the payload is appended at EOF. If the /// directory is full the directory grows by one 32-byte slot: every file /// payload shifts forward by 32 bytes, every existing slot's absolute /// dataOffset field is patched by +32, then the new slot is written and -/// the new payload appended at the new EOF. Header's maxEntries and -/// usedEntries are updated to match. -/// Remove: shifts the later directory slots up by 32 bytes inside -/// the directory table (the vacated trailing slot is zero-filled to leave no -/// forensic trace), then wipes the removed file's payload bytes and shifts the -/// remaining payload region into the vacated payload range (updating each -/// affected slot's dataOffset field). The stream is truncated to the -/// new EOF. Both maxEntries and usedEntries are decremented so -/// the directory remains exactly sized. +/// the new payload appended at the new EOF. +/// Remove: shifts the later directory slots up by 32 bytes, wipes +/// the removed payload, closes the payload gap, patches all surviving absolute +/// data offsets, and truncates the stream. /// public static class T64InPlaceModifier { private const int HeaderSize = 64; private const int EntrySize = 32; private const int MaxEntriesOffset = 34; - private const int UsedEntriesOffset = 36; /// /// Adds (or replaces by name, case-insensitive) a single file inside an @@ -47,55 +41,57 @@ public static void AddFile(Stream image, string name, byte[] data, ushort startA ArgumentNullException.ThrowIfNull(image); ArgumentNullException.ThrowIfNull(name); ArgumentNullException.ThrowIfNull(data); - if (!image.CanRead || !image.CanWrite || !image.CanSeek) - throw new ArgumentException("T64: stream must be readable, writable and seekable.", nameof(image)); + EnsureMutable(image); + ValidatePayload(name, startAddress, data.Length); + + // Validate the image before replacement semantics can remove anything. + image.Position = 0; + using (var reader = new T64Reader(image)) { } - // Replace-by-name semantic: drop any prior entry with the same name first. var existingIndex = FindEntryIndex(image, name); if (existingIndex >= 0) RemoveEntryAt(image, existingIndex); - // Re-read header after possible removal. - var (maxEntries, usedEntries) = ReadHeaderCounts(image); + var (maxEntries, _) = ReadHeaderCounts(image); + var usedEntries = CountUsedEntries(image, maxEntries); - // Try to find a free slot first. var freeSlot = FindFreeSlot(image, maxEntries); if (freeSlot >= 0) { AppendDataAndFillSlot(image, freeSlot, name, data, startAddress); - WriteHeaderCounts(image, maxEntries, (ushort)(usedEntries + 1)); + WriteHeaderCounts(image, maxEntries, checked((ushort)(usedEntries + 1))); return; } - // Directory is full → grow by one slot. Shift payload region right by 32 - // bytes and patch every existing slot's absolute dataOffset by +32. + if (maxEntries == ushort.MaxValue) + throw new IOException("T64: directory cannot grow beyond 65535 slots."); + + // Directory is full: insert one slot immediately before the payload region. var payloadStart = (long)HeaderSize + maxEntries * EntrySize; var payloadLength = image.Length - payloadStart; ShiftRangeForward(image, payloadStart, payloadLength, EntrySize); - // Zero-init the new slot's bytes so any future scan sees a clean slot. Span zero = stackalloc byte[EntrySize]; image.Position = payloadStart; image.Write(zero); - // Patch every existing slot's absolute dataOffset += EntrySize. Span entry = stackalloc byte[EntrySize]; for (var i = 0; i < maxEntries; i++) { var slotOff = HeaderSize + i * EntrySize; image.Position = slotOff; image.ReadExactly(entry); if (entry[0] == 0) continue; - var oldDataOff = BinaryPrimitives.ReadUInt32LittleEndian(entry.Slice(8)); - BinaryPrimitives.WriteUInt32LittleEndian(entry.Slice(8), oldDataOff + (uint)EntrySize); + var oldDataOff = BinaryPrimitives.ReadUInt32LittleEndian(entry[8..]); + if (oldDataOff > uint.MaxValue - EntrySize) + throw new InvalidDataException("T64: payload offset cannot be shifted into the 32-bit offset field."); + BinaryPrimitives.WriteUInt32LittleEndian(entry[8..], oldDataOff + EntrySize); image.Position = slotOff; image.Write(entry); } - // Write new slot at the freed gap (the now-zeroed bytes at payloadStart). var newSlotIndex = maxEntries; - var newMaxEntries = (ushort)(maxEntries + 1); + var newMaxEntries = checked((ushort)(maxEntries + 1)); AppendDataAndFillSlot(image, newSlotIndex, name, data, startAddress); - - WriteHeaderCounts(image, newMaxEntries, (ushort)(usedEntries + 1)); + WriteHeaderCounts(image, newMaxEntries, checked((ushort)(usedEntries + 1))); } /// @@ -105,8 +101,10 @@ public static void AddFile(Stream image, string name, byte[] data, ushort startA public static bool RemoveFile(Stream image, string name) { ArgumentNullException.ThrowIfNull(image); ArgumentNullException.ThrowIfNull(name); - if (!image.CanRead || !image.CanWrite || !image.CanSeek) - throw new ArgumentException("T64: stream must be readable, writable and seekable.", nameof(image)); + EnsureMutable(image); + + image.Position = 0; + using (var reader = new T64Reader(image)) { } var index = FindEntryIndex(image, name); if (index < 0) return false; @@ -114,26 +112,20 @@ public static bool RemoveFile(Stream image, string name) { return true; } - // ── Internals ──────────────────────────────────────────────────────────── - private static void RemoveEntryAt(Stream image, int index) { - var (maxEntries, usedEntries) = ReadHeaderCounts(image); + var (maxEntries, _) = ReadHeaderCounts(image); if (index < 0 || index >= maxEntries) return; - // Read the slot we're removing to learn the data extent. - Span entry = stackalloc byte[EntrySize]; - var slotOff = HeaderSize + index * EntrySize; - image.Position = slotOff; - image.ReadExactly(entry); - - var startAddr = BinaryPrimitives.ReadUInt16LittleEndian(entry.Slice(2)); - var endAddr = BinaryPrimitives.ReadUInt16LittleEndian(entry.Slice(4)); - var removedDataOffset = (long)BinaryPrimitives.ReadUInt32LittleEndian(entry.Slice(8)); - var removedDataLen = endAddr > startAddr ? endAddr - startAddr : 0; - var wasOccupied = entry[0] != 0; - - // Wipe the removed payload bytes for the forensic guarantee. - if (wasOccupied && removedDataLen > 0) + image.Position = 0; + using var reader = new T64Reader(image); + var removed = reader.Entries.FirstOrDefault(e => e.DirectoryIndex == index); + if (removed is null) return; + + var removedDataOffset = (long)removed.DataOffset; + var removedDataLen = removed.Size; + var usedEntries = reader.Entries.Count; + + if (removedDataLen > 0) ZeroRange(image, removedDataOffset, removedDataLen); // 1) Compact directory: shift later slots up by 32 bytes. @@ -142,79 +134,66 @@ private static void RemoveEntryAt(Stream image, int index) { if (laterSlotsLen > 0) ShiftRangeBackward(image, afterEntryOffset, laterSlotsLen, EntrySize); - // 2) Compact payload region: the directory shrank by 32 bytes, so all file - // payloads shift back by 32. Additionally, the removed payload's bytes - // must be excised. - // Old payload region: [oldPayloadStart .. image.Length) - // Within it, removedDataOffset .. removedDataOffset+removedDataLen is gone. - // New payload region: shifted back by 32 bytes, and the removed slice is closed up. + // 2) Compact payload bytes. Every payload loses the 32-byte directory slot; + // payloads after the removed member additionally lose that member's bytes. var oldPayloadStart = (long)HeaderSize + maxEntries * EntrySize; - var newPayloadStart = oldPayloadStart - EntrySize; var oldEnd = image.Length; - // Sub-region A: [oldPayloadStart .. removedDataOffset) shifts back by 32. - var aLen = (wasOccupied ? removedDataOffset : oldEnd) - oldPayloadStart; - if (aLen > 0) - ShiftRangeBackward(image, oldPayloadStart, aLen, EntrySize); - - // Sub-region B: [removedDataOffset+removedDataLen .. oldEnd) shifts back by - // (32 + removedDataLen) so the removed-payload slice closes up. - var bSrc = removedDataOffset + removedDataLen; - var bLen = oldEnd - bSrc; - if (wasOccupied && bLen > 0) - ShiftRangeBackward(image, bSrc, bLen, EntrySize + removedDataLen); - - // 3) Patch every remaining slot's absolute dataOffset. - // Slots whose dataOffset was < removedDataOffset shift by -32. - // Slots whose dataOffset was > removedDataOffset shift by -(32 + removedDataLen). - // (The removed slot itself is already gone after directory compaction.) - var newMaxEntries = (ushort)(maxEntries - 1); + var beforeRemovedLength = removedDataOffset - oldPayloadStart; + if (beforeRemovedLength > 0) + ShiftRangeBackward(image, oldPayloadStart, beforeRemovedLength, EntrySize); + + var afterRemovedSource = checked(removedDataOffset + removedDataLen); + var afterRemovedLength = oldEnd - afterRemovedSource; + if (afterRemovedLength > 0) + ShiftRangeBackward(image, afterRemovedSource, afterRemovedLength, EntrySize + removedDataLen); + + // 3) Patch surviving absolute payload offsets after the directory move. + Span entry = stackalloc byte[EntrySize]; + var newMaxEntries = checked((ushort)(maxEntries - 1)); for (var i = 0; i < newMaxEntries; i++) { - var so = HeaderSize + i * EntrySize; - image.Position = so; + var slotOff = HeaderSize + i * EntrySize; + image.Position = slotOff; image.ReadExactly(entry); if (entry[0] == 0) continue; - var dataOff = (long)BinaryPrimitives.ReadUInt32LittleEndian(entry.Slice(8)); - long newDataOff; - if (!wasOccupied || dataOff < removedDataOffset) - newDataOff = dataOff - EntrySize; - else - newDataOff = dataOff - EntrySize - removedDataLen; - BinaryPrimitives.WriteUInt32LittleEndian(entry.Slice(8), (uint)newDataOff); - image.Position = so; + + var dataOff = (long)BinaryPrimitives.ReadUInt32LittleEndian(entry[8..]); + var newDataOff = dataOff < removedDataOffset + ? dataOff - EntrySize + : dataOff - EntrySize - removedDataLen; + if (newDataOff < HeaderSize + (long)newMaxEntries * EntrySize || newDataOff > uint.MaxValue) + throw new InvalidDataException("T64: compaction produced an invalid payload offset."); + + BinaryPrimitives.WriteUInt32LittleEndian(entry[8..], (uint)newDataOff); + image.Position = slotOff; image.Write(entry); } - // 4) Truncate the stream to the new EOF. - var newEnd = oldEnd - EntrySize - (wasOccupied ? removedDataLen : 0); + // 4) Truncate and repair both header counts from the actual live directory. + var newEnd = oldEnd - EntrySize - removedDataLen; image.SetLength(newEnd); - - // 5) Patch header counts. - var newUsedEntries = wasOccupied && usedEntries > 0 ? (ushort)(usedEntries - 1) : usedEntries; - WriteHeaderCounts(image, newMaxEntries, newUsedEntries); + WriteHeaderCounts(image, newMaxEntries, checked((ushort)(usedEntries - 1))); } - /// - /// Writes data at EOF, then fills the given directory slot with type, - /// addresses, dataOffset and filename. - /// private static void AppendDataAndFillSlot(Stream image, int slotIndex, string name, byte[] data, ushort startAddress) { var dataOffset = image.Length; + if (dataOffset > uint.MaxValue) + throw new IOException("T64: payload offset exceeds the 32-bit directory field."); + image.Position = dataOffset; image.Write(data); Span entry = stackalloc byte[EntrySize]; - entry[0] = 1; // entry type: normal - entry[1] = 0x82; // C64 file type: PRG - BinaryPrimitives.WriteUInt16LittleEndian(entry.Slice(2), startAddress); - BinaryPrimitives.WriteUInt16LittleEndian(entry.Slice(4), (ushort)(startAddress + data.Length)); - BinaryPrimitives.WriteUInt32LittleEndian(entry.Slice(8), (uint)dataOffset); + entry[0] = 1; + entry[1] = 0x82; + BinaryPrimitives.WriteUInt16LittleEndian(entry[2..], startAddress); + BinaryPrimitives.WriteUInt16LittleEndian(entry[4..], unchecked((ushort)(startAddress + data.Length))); + BinaryPrimitives.WriteUInt32LittleEndian(entry[8..], (uint)dataOffset); var trimmed = name.Length > 16 ? name[..16] : name; var nameBytes = Encoding.ASCII.GetBytes(trimmed); - nameBytes.CopyTo(entry.Slice(16)); - for (var j = nameBytes.Length; j < 16; j++) - entry[16 + j] = 0x20; + nameBytes.CopyTo(entry[16..]); + entry[(16 + nameBytes.Length)..32].Fill(0x20); image.Position = HeaderSize + slotIndex * EntrySize; image.Write(entry); @@ -223,8 +202,7 @@ private static void AppendDataAndFillSlot(Stream image, int slotIndex, string na private static int FindFreeSlot(Stream image, int maxEntries) { for (var i = 0; i < maxEntries; i++) { image.Position = HeaderSize + i * EntrySize; - var typeByte = image.ReadByte(); - if (typeByte == 0) return i; + if (image.ReadByte() == 0) return i; } return -1; } @@ -237,101 +215,114 @@ private static int FindEntryIndex(Stream image, string name) { image.Position = HeaderSize + i * EntrySize; image.ReadExactly(entry); if (entry[0] == 0) continue; - var entryName = Encoding.ASCII.GetString(entry.Slice(16, 16)).TrimEnd('\0', ' '); + var entryName = Encoding.ASCII.GetString(entry[16..32]).TrimEnd('\0', ' '); if (entryName.Equals(trimmed, StringComparison.OrdinalIgnoreCase)) return i; } return -1; } + private static int CountUsedEntries(Stream image, int maxEntries) { + var result = 0; + for (var i = 0; i < maxEntries; i++) { + image.Position = HeaderSize + i * EntrySize; + if (image.ReadByte() > 0) ++result; + } + return result; + } + private static (ushort MaxEntries, ushort UsedEntries) ReadHeaderCounts(Stream image) { if (image.Length < HeaderSize) throw new InvalidDataException("T64: stream too small."); - Span buf = stackalloc byte[4]; + Span buffer = stackalloc byte[4]; image.Position = MaxEntriesOffset; - image.ReadExactly(buf); - return ( - BinaryPrimitives.ReadUInt16LittleEndian(buf), - BinaryPrimitives.ReadUInt16LittleEndian(buf.Slice(2))); + image.ReadExactly(buffer); + var maxEntries = BinaryPrimitives.ReadUInt16LittleEndian(buffer); + var usedEntries = BinaryPrimitives.ReadUInt16LittleEndian(buffer[2..]); + if ((long)HeaderSize + maxEntries * EntrySize > image.Length) + throw new InvalidDataException("T64: directory extends beyond end of image."); + return (maxEntries, usedEntries); } private static void WriteHeaderCounts(Stream image, ushort maxEntries, ushort usedEntries) { - Span buf = stackalloc byte[4]; - BinaryPrimitives.WriteUInt16LittleEndian(buf, maxEntries); - BinaryPrimitives.WriteUInt16LittleEndian(buf.Slice(2), usedEntries); + Span buffer = stackalloc byte[4]; + BinaryPrimitives.WriteUInt16LittleEndian(buffer, maxEntries); + BinaryPrimitives.WriteUInt16LittleEndian(buffer[2..], usedEntries); image.Position = MaxEntriesOffset; - image.Write(buf); + image.Write(buffer); + } + + private static void EnsureMutable(Stream image) { + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("T64: stream must be readable, writable and seekable.", nameof(image)); + } + + private static void ValidatePayload(string name, ushort startAddress, int length) { + if (startAddress + (long)length > 0x10000) + throw new InvalidOperationException( + $"T64: '{name}' is {length:N0} bytes and loads at ${startAddress:X4}, past the C64 64 KB address space."); } private static void ZeroRange(Stream image, long offset, long length) { if (length <= 0) return; - var buf = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); + var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); try { - Array.Clear(buf); + Array.Clear(buffer, 0, buffer.Length); var remaining = length; image.Position = offset; while (remaining > 0) { - var chunk = (int)Math.Min(remaining, buf.Length); - image.Write(buf, 0, chunk); + var chunk = (int)Math.Min(remaining, buffer.Length); + image.Write(buffer, 0, chunk); remaining -= chunk; } } finally { - ArrayPool.Shared.Return(buf); + ArrayPool.Shared.Return(buffer); } } - /// - /// Shifts bytes [src .. src+length) to [src+delta .. src+delta+length), where - /// delta > 0. Copies high-to-low so overlap doesn't corrupt the data. - /// private static void ShiftRangeForward(Stream image, long src, long length, long delta) { if (length <= 0 || delta == 0) return; if (delta < 0) throw new ArgumentOutOfRangeException(nameof(delta)); - var dstEnd = src + delta + length; + var dstEnd = checked(src + delta + length); if (dstEnd > image.Length) image.SetLength(dstEnd); - var buf = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); + var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); try { var remaining = length; while (remaining > 0) { - var chunk = (int)Math.Min(remaining, buf.Length); + var chunk = (int)Math.Min(remaining, buffer.Length); var readFrom = src + remaining - chunk; - var writeTo = readFrom + delta; image.Position = readFrom; - image.ReadExactly(buf, 0, chunk); - image.Position = writeTo; - image.Write(buf, 0, chunk); + image.ReadExactly(buffer, 0, chunk); + image.Position = readFrom + delta; + image.Write(buffer, 0, chunk); remaining -= chunk; } } finally { - ArrayPool.Shared.Return(buf); + ArrayPool.Shared.Return(buffer); } } - /// - /// Shifts bytes [src .. src+length) to [src-delta .. src-delta+length), where - /// delta > 0. Copies low-to-high since destination < source. - /// private static void ShiftRangeBackward(Stream image, long src, long length, long delta) { if (length <= 0 || delta == 0) return; if (delta < 0) throw new ArgumentOutOfRangeException(nameof(delta)); - var buf = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); + var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); try { var remaining = length; var cursor = 0L; while (remaining > 0) { - var chunk = (int)Math.Min(remaining, buf.Length); + var chunk = (int)Math.Min(remaining, buffer.Length); image.Position = src + cursor; - image.ReadExactly(buf, 0, chunk); + image.ReadExactly(buffer, 0, chunk); image.Position = src + cursor - delta; - image.Write(buf, 0, chunk); + image.Write(buffer, 0, chunk); cursor += chunk; remaining -= chunk; } } finally { - ArrayPool.Shared.Return(buf); + ArrayPool.Shared.Return(buffer); } } } From 74cf7a3f7976551535e28684fc2985a290f374fc Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:02:33 +0200 Subject: [PATCH 039/165] Preserve 4 GiB segment endpoints in firmware metadata --- FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs b/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs index a7ea0150d..a2cc08628 100644 --- a/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs +++ b/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs @@ -47,8 +47,9 @@ private static byte[] BuildMetadata(FirmwareImage image) { } for (var i = 0; i < image.Segments.Count; i++) { var (a, d) = image.Segments[i]; + var end = (ulong)a + (uint)d.Length; sb.Append(CultureInfo.InvariantCulture, - $"segment_{i} = 0x{a:X8} .. 0x{a + (uint)d.Length:X8} ({d.Length} bytes)\n"); + $"segment_{i} = 0x{a:X8} .. 0x{end:X8} ({d.Length} bytes)\n"); } return Encoding.UTF8.GetBytes(sb.ToString()); } From 1ed681706612f8860b7333dba4cc8d712de6f8ab Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:02:50 +0200 Subject: [PATCH 040/165] + add TI-TXT read-write and maintenance operations --- .../TiTxtFormatDescriptor.cs | 242 +++++++++++++----- 1 file changed, 177 insertions(+), 65 deletions(-) diff --git a/FileFormats/FileFormat.FirmwareHex/TiTxtFormatDescriptor.cs b/FileFormats/FileFormat.FirmwareHex/TiTxtFormatDescriptor.cs index 79c244c7c..0b83df390 100644 --- a/FileFormats/FileFormat.FirmwareHex/TiTxtFormatDescriptor.cs +++ b/FileFormats/FileFormat.FirmwareHex/TiTxtFormatDescriptor.cs @@ -1,4 +1,5 @@ #pragma warning disable CS1591 +using System.Globalization; using System.Text; using Compression.Registry; using static Compression.Registry.FormatHelpers; @@ -7,84 +8,58 @@ namespace FileFormat.FirmwareHex; /// /// Pseudo-archive descriptor for the TI-TXT firmware text format used by MSP430. -/// Address lines (@HHHH) introduce contiguous byte runs; a single q -/// terminates the file. Extension is intentionally empty — .txt is far -/// too ambiguous — so detection relies on the first non-whitespace byte being -/// @. +/// Address lines (@HHHH) introduce sparse byte runs; a line containing +/// q terminates the file. The archive view exposes the logical firmware as +/// firmware.bin plus a rendered metadata.ini summary. /// /// References: /// -/// Texas Instruments MSP430 programming/bootloader guides — define the TI-TXT format (@addr / data / q) -/// https://srecord.sourceforge.net — SRecord tool suite — documents and converts TI-TXT (srec_ti_txt) +/// https://downloads.ti.com/docs/esd/SPRU513P/Content/SPRU513P_HTML/hex-conversion-utility-description.html — TI Hex Conversion Utility, TI-TXT format +/// https://www.ti.com/lit/pdf/slau131 — Texas Instruments MSP430 programming documentation +/// https://srecord.sourceforge.net/man/man5/srec_ti_txt.5.html — SRecord TI-TXT documentation; used only as an interoperability reference /// /// -public sealed class TiTxtFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable { +public sealed class TiTxtFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, + IArchiveModifiable, IArchiveDefragmentable, IArchiveWriteConstraints, ISyntheticEntryNames { + + private static readonly IReadOnlySet SyntheticNames = + new HashSet(StringComparer.OrdinalIgnoreCase) { FirmwareHexWriter.MetadataName }; - /// - /// Gets the id. - /// public string Id => "TiTxt"; - /// - /// Gets the display name. - /// public string DisplayName => "TI-TXT (MSP430)"; - /// - /// Gets the category. - /// public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | - FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; - /// - /// Gets the default extension. - /// + FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; public string DefaultExtension => ".txt"; - /// - /// Gets the extensions. - /// public IReadOnlyList Extensions => []; - /// - /// Gets the compound extensions. - /// public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// - public IReadOnlyList MagicSignatures => [ - // TI-TXT begins with '@' on the first non-whitespace line; low confidence - // because '@' shows up in lots of text formats (email/yaml/etc). - new([(byte)'@'], Confidence: 0.15), - ]; - /// - /// Gets the methods. - /// + public IReadOnlyList MagicSignatures => [new([(byte)'@'], Confidence: 0.15)]; public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// public AlgorithmFamily Family => AlgorithmFamily.Encoding; - /// - /// Gets the description. - /// public string Description => - "TI-TXT MSP430 firmware text (address/data/q-terminator)."; + "TI-TXT MSP430 sparse firmware text; supports create, replace/remove, purge and canonical rebuild."; + + public IReadOnlySet SyntheticEntryNames => SyntheticNames; + + public bool CanAccept(ArchiveInputInfo input, out string? reason) { + ArgumentNullException.ThrowIfNull(input); + if (!input.IsDirectory) { + reason = null; + return true; + } + + reason = "TI-TXT accepts one firmware payload plus optional metadata.ini; directories are not representable."; + return false; + } + + public long? MaxTotalArchiveSize => null; + public string AcceptedInputsDescription => "one firmware payload plus optional metadata.ini"; - /// - /// Lists the entries in the supplied container. - /// public List List(Stream stream, string? password) => FirmwareHexCommon.BuildArchiveEntries(BuildEntries(stream)); - /// - /// Decodes the supplied input. - /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { foreach (var e in BuildEntries(stream)) { if (files != null && files.Length > 0 && !MatchesFilter(e.Name, files)) continue; @@ -92,20 +67,157 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? } } - /// - /// Writes a fresh TI-TXT file: the single payload input becomes the data lines - /// under an @address taken from a metadata.ini alongside it, and - /// the file ends with the q the format requires. - /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + ValidateInputs(inputs, out _, out _); FirmwareHexWriter.WriteTiTxt(output, FirmwareHexWriter.ImageFrom(inputs, "TiTxt")); } - private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) { + /// + /// Replaces the single logical firmware payload and/or moves its base address. + /// A metadata-only base-address edit shifts every sparse section by the same + /// delta so address holes remain holes rather than becoming explicit 0xFF data. + /// + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + ValidateInputs(inputs, out var payload, out var metadata); + if (payload is null && metadata is null) return; + + var current = ReadImage(archive); + var baseAddress = metadata is null ? current.BaseAddress : ReadBaseAddress(metadata.ReadContent()) ?? current.BaseAddress; + FirmwareImage replacement; + + if (payload is not null) { + var bytes = payload.ReadContent(); + replacement = ImageFromSegments( + bytes.Length == 0 ? [] : [(baseAddress, bytes)]); + } else if (current.Segments.Count == 0 || baseAddress == current.BaseAddress) { + replacement = current; + } else { + replacement = ShiftImage(current, baseAddress); + } + + Rewrite(archive, replacement); + } + + /// Removing the logical firmware payload leaves the valid empty TI-TXT form, q. + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryNames); + if (!entryNames.Any(name => Path.GetFileName(name).Equals(FirmwareHexWriter.PayloadName, StringComparison.OrdinalIgnoreCase))) + return; + Rewrite(archive, ImageFromSegments([])); + } + + /// Erases all live firmware bytes while leaving a valid empty TI-TXT document. + public void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + _ = ReadImage(archive); // reject malformed input before replacing it + Rewrite(archive, ImageFromSegments([])); + } + + /// + /// Canonicalises section ordering and record wrapping without flattening sparse + /// holes. The reader merges only exactly adjacent sections; the writer then + /// emits one address record per contiguous run and 16 bytes per full data line. + /// + public void Defragment(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + Rewrite(archive, ReadImage(archive)); + } + + private static void ValidateInputs(IReadOnlyList inputs, + out ArchiveInputInfo? payload, out ArchiveInputInfo? metadata) { + payload = null; + metadata = null; + foreach (var input in inputs) { + if (input.IsDirectory) + throw new InvalidDataException("TI-TXT cannot store directories."); + if (Path.GetFileName(input.ArchiveName).Equals(FirmwareHexWriter.MetadataName, StringComparison.OrdinalIgnoreCase)) { + if (metadata is not null) + throw new InvalidDataException("TI-TXT accepts at most one metadata.ini input."); + metadata = input; + continue; + } + if (payload is not null) + throw new InvalidDataException("TI-TXT represents one logical firmware payload; multiple payload inputs are ambiguous."); + payload = input; + } + } + + private static uint? ReadBaseAddress(byte[] metadata) { + foreach (var raw in Encoding.UTF8.GetString(metadata).Split('\n')) { + var line = raw.Trim(); + var equals = line.IndexOf('=', StringComparison.Ordinal); + if (equals < 0 || !line[..equals].Trim().Equals("base_address", StringComparison.OrdinalIgnoreCase)) continue; + var value = line[(equals + 1)..].Trim(); + if (value.StartsWith("0x", StringComparison.OrdinalIgnoreCase) && + uint.TryParse(value.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var parsed)) + return parsed; + } + return null; + } + + private static FirmwareImage ShiftImage(FirmwareImage image, uint newBaseAddress) { + var delta = (long)newBaseAddress - image.BaseAddress; + var shifted = new List<(uint Address, byte[] Data)>(image.Segments.Count); + foreach (var (address, data) in image.Segments) { + var target = (long)address + delta; + if (target < 0 || target > uint.MaxValue || (ulong)target + (ulong)data.LongLength > (ulong)uint.MaxValue + 1) + throw new InvalidDataException("TI-TXT base-address edit would move a section outside the 32-bit address space."); + shifted.Add(((uint)target, data)); + } + return ImageFromSegments(shifted); + } + + private static FirmwareImage ImageFromSegments(List<(uint Address, byte[] Data)> segments) { + var totalBytes = 0; + for (var i = 0; i < segments.Count; ++i) checked { totalBytes += segments[i].Data.Length; } + var gaps = 0; + for (var i = 1; i < segments.Count; ++i) + if ((ulong)segments[i - 1].Address + (ulong)segments[i - 1].Data.Length < segments[i].Address) gaps++; + return new FirmwareImage(segments, StartAddress: null, RecordCount: 0, GapCount: gaps, + TotalDataBytes: totalBytes, SourceFormat: "TiTxt"); + } + + private static FirmwareImage ReadImage(Stream stream) { + if (!stream.CanRead) throw new ArgumentException("TI-TXT stream must be readable.", nameof(stream)); + if (stream.CanSeek) stream.Position = 0; using var reader = new StreamReader(stream, Encoding.ASCII, detectEncodingFromByteOrderMarks: true, leaveOpen: true); - var text = reader.ReadToEnd(); - var image = TiTxtReader.Read(text); - return FirmwareHexCommon.BuildEntries(image); + return TiTxtReader.Read(reader.ReadToEnd()); + } + + private static void Rewrite(Stream archive, FirmwareImage image) { + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("TI-TXT mutation requires a readable, writable, seekable stream.", nameof(archive)); + + using var staged = new MemoryStream(); + FirmwareHexWriter.WriteTiTxt(staged, image); + staged.Position = 0; + var reparsed = ReadImage(staged); + if (!SameSparseImage(image, reparsed)) + throw new InvalidDataException("TI-TXT rewrite verification failed: sparse address/data semantics changed."); + + archive.Position = 0; + archive.SetLength(0); + staged.Position = 0; + staged.CopyTo(archive); + archive.SetLength(archive.Position); + archive.Position = 0; } + + private static bool SameSparseImage(FirmwareImage expected, FirmwareImage actual) { + if (expected.Segments.Count != actual.Segments.Count) return false; + for (var i = 0; i < expected.Segments.Count; ++i) { + var left = expected.Segments[i]; + var right = actual.Segments[i]; + if (left.Address != right.Address || !left.Data.AsSpan().SequenceEqual(right.Data)) return false; + } + return true; + } + + private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) => + FirmwareHexCommon.BuildEntries(ReadImage(stream)); } From 4f0dd67192e198c7904d02e3dc61ff8cc7c1859f Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:03:00 +0200 Subject: [PATCH 041/165] Keep Lynx tests on public surface --- Compression.Tests/Lynx/LynxTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Compression.Tests/Lynx/LynxTests.cs b/Compression.Tests/Lynx/LynxTests.cs index 7eb4825de..667bb3c97 100644 --- a/Compression.Tests/Lynx/LynxTests.cs +++ b/Compression.Tests/Lynx/LynxTests.cs @@ -191,7 +191,7 @@ public void Shrink_ReclaimsDirectoryOverallocationAndTrailer_WithoutChangingPayl Assert.Multiple(() => { Assert.That(shrunk.Length, Is.LessThan(bloatedLength)); - Assert.That(shrunk.Length % LynxReader.BlockSize, Is.Zero); + Assert.That(shrunk.Length % 254, Is.Zero); Assert.That(ReadDirectoryBlockCount(shrunk.ToArray()), Is.EqualTo(1)); }); shrunk.Position = 0; From bacfde094f402734a51277b3623d282f71f7f447 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:03:05 +0200 Subject: [PATCH 042/165] Handle 4 GiB Intel HEX segment endpoints safely --- .../FileFormat.FirmwareHex/FirmwareHexWriter.cs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs b/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs index 77d50eb18..6c7d4b452 100644 --- a/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs +++ b/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs @@ -122,6 +122,12 @@ private static bool TryParseHexUInt16(string text, out ushort value) { && ushort.TryParse(text.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out value); } + private static bool TryParseHexUInt64(string text, out ulong value) { + value = 0; + return text.StartsWith("0x", StringComparison.OrdinalIgnoreCase) + && ulong.TryParse(text.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out value); + } + private static bool TryParseSegmentLayout(string value, out SegmentLayout segment) { segment = default; const string separator = " .. "; @@ -133,10 +139,11 @@ private static bool TryParseSegmentLayout(string value, out SegmentLayout segmen var endTokenLength = remainder.IndexOf(' '); if (endTokenLength < 0) endTokenLength = remainder.Length; var endText = remainder[..endTokenLength]; - if (!TryParseHexUInt32(startText, out var start) || !TryParseHexUInt32(endText, out var end) || end < start) + if (!TryParseHexUInt32(startText, out var start) || !TryParseHexUInt64(endText, out var end) + || end < start || end > 0x1_0000_0000UL) return false; - var length = (ulong)end - start; + var length = end - start; if (length > int.MaxValue) return false; segment = new SegmentLayout(start, (int)length); return true; @@ -159,7 +166,7 @@ private static bool TryRestoreSparseSegments(byte[] payload, uint baseAddress, ulong previousEnd = baseAddress; ulong highestEnd = baseAddress; foreach (var segment in ordered) { - if (segment.Length < 0 || segment.Address < previousEnd) return false; + if (segment.Address < previousEnd) return false; var end = (ulong)segment.Address + (uint)segment.Length; if (end > 0x1_0000_0000UL) return false; previousEnd = end; @@ -170,7 +177,7 @@ private static bool TryRestoreSparseSegments(byte[] payload, uint baseAddress, foreach (var segment in ordered) { var offset = (ulong)segment.Address - baseAddress; if (offset + (uint)segment.Length > (ulong)payload.Length) return false; - segments.Add((segment.Address, payload.AsSpan((int)offset, segment.Length).ToArray())); + segments.Add((segment.Address, payload.AsSpan(checked((int)offset), segment.Length).ToArray())); } return true; } From 5e9499461baadeb0b815b75b0f48534986544537 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:03:10 +0200 Subject: [PATCH 043/165] fix(onefs): make opaque inspection streaming and fail closed --- FileSystems/FileSystem.OneFs/OneFsReader.cs | 216 +++++++++++++------- 1 file changed, 139 insertions(+), 77 deletions(-) diff --git a/FileSystems/FileSystem.OneFs/OneFsReader.cs b/FileSystems/FileSystem.OneFs/OneFsReader.cs index d97cf5aad..ea6c6d8e0 100644 --- a/FileSystems/FileSystem.OneFs/OneFsReader.cs +++ b/FileSystems/FileSystem.OneFs/OneFsReader.cs @@ -1,120 +1,182 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; using System.Globalization; using System.Text; +using Compression.Registry.Streaming; namespace FileSystem.OneFs; /// -/// Stage 0 detection-only reader for Dell EMC Isilon OneFS LIN-tree root -/// images. OneFS is a clustered scale-out NAS — its single-image surface -/// is the LIN-tree root block, whose first bytes are the ASCII tag -/// "OneFS" (5 bytes, 0x4F 0x6E 0x65 0x46 0x53) or the short -/// "ONEF" tag (0x4F 0x4E 0x45 0x46 = 0x4F4E4546 BE int) used in -/// some node-local boot images. -/// +/// Conservative single-image inspection surface for Dell PowerScale / Isilon +/// OneFS media. +/// +/// /// -/// Only the tag is verified; the real LIN tree (logical inode number tree) -/// is a cluster-wide construct and cannot be walked from a single image. -/// File data is FEC-striped across nodes (N+M:B protection groups) — even -/// a complete single-drive image carries only one stripe and cannot -/// reconstruct file content without peer nodes. +/// Dell documents the cluster-level structures required to resolve OneFS data: +/// the superblock points at the LIN master, the LIN B+ tree maps logical inode +/// numbers to mirrored inode addresses, and file metatrees map logical blocks to +/// protection groups. Those addresses identify a node, drive and physical block, +/// so a lone drive image is not a self-contained filesystem namespace. /// /// -/// OneFS shares OS ancestry with FreeBSD, but the on-disk filesystem layer -/// is proprietary and NOT UFS-compatible: there is no UFS1 superblock magic -/// (0x00011954) at the UFS1 superblock offset (8192). The OneFS -/// on-disk format has never been publicly specified by Dell EMC. +/// Dell does not publish a fixed raw-image magic value that identifies a OneFS +/// drive at offset zero. In particular, the historical "OneFS" / "ONEF" +/// literals previously used here could not be corroborated by Dell documentation +/// and are therefore not parsed or advertised as signatures. The reader is +/// reached by explicit format selection or the .onefs extension and treats +/// the supplied bytes as opaque media. /// -/// +/// +/// The reader deliberately performs no payload reads while listing. This keeps a +/// multi-terabyte disk image out of managed memory and, more importantly, avoids +/// inventing structure from undocumented bytes. The raw image remains available +/// as a bounded streaming entry for forensic inspection or export. +/// +/// +/// References used for the clean-room behaviour documented here: +/// +/// Dell Technologies Info Hub, "OneFS Metadata" (LIN tree, +/// inode mirrors, IFM/DFM B+ trees and protection groups). +/// Dell PowerScale OneFS Technical Overview, "File system +/// structure" (distributed UFS-based filesystem and single namespace). +/// Dell PowerScale OneFS Technical Specifications Guide +/// (8 KiB filesystem block size). +/// +/// No external implementation code is copied or translated. +/// +/// public sealed class OneFsReader : IDisposable { - /// OneFS LIN-tree-root long tag: ASCII "OneFS" (5 bytes). - public static readonly byte[] LongTag = "OneFS"u8.ToArray(); - /// OneFS short tag: ASCII "ONEF" (4 bytes, 0x4F4E4546 BE). - public static readonly byte[] ShortTag = "ONEF"u8.ToArray(); + /// Documented OneFS filesystem block size. + public const int PhysicalBlockSize = 8 * 1024; + + /// Name of the synthetic inspection metadata entry. + public const string MetadataEntryName = "metadata.ini"; - private readonly byte[] _data; + /// Name of the opaque raw-image entry. + public const string RawImageEntryName = "onefs-volume.bin"; + + private readonly Stream _stream; + private readonly bool _leaveOpen; private readonly List _entries = []; + /// Gets the synthetic entries exposed by the conservative reader. + public IReadOnlyList Entries => this._entries; + /// - /// Gets the entries. - /// - public IReadOnlyList Entries => _entries; - /// - /// Gets or sets the tag. + /// Legacy compatibility property. No authoritative fixed OneFS raw-media tag + /// is currently known, so this value is always empty. /// public string Tag { get; private set; } = ""; + /// - /// Gets or sets the trailing word. + /// Legacy compatibility property. No undocumented trailing header word is + /// interpreted; this value is always zero. /// public uint TrailingWord { get; private set; } + /// - /// Gets a value indicating whether valid header. + /// Legacy compatibility property. The current reader does not claim to have + /// validated a proprietary raw-media header, so this value is always false. /// public bool ValidHeader { get; private set; } + /// Gets the byte length of the opaque source image. + public long ImageSize { get; } + /// - /// Initializes a new instance of . + /// Initializes a conservative OneFS image reader without consuming the image. /// - public OneFsReader(Stream stream) { - using var ms = new MemoryStream(); - stream.CopyTo(ms); - _data = ms.ToArray(); - Parse(); + /// Readable, seekable raw image stream. + /// Whether disposing the reader leaves the source open. + public OneFsReader(Stream stream, bool leaveOpen = true) { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead) + throw new ArgumentException("OneFS inspection requires a readable stream.", nameof(stream)); + if (!stream.CanSeek) + throw new ArgumentException("OneFS inspection requires a seekable stream.", nameof(stream)); + if (stream.Length <= 0) + throw new InvalidDataException("OneFS: empty image cannot be inspected."); + + this._stream = stream; + this._leaveOpen = leaveOpen; + this.ImageSize = stream.Length; + this.BuildEntries(); } - private void Parse() { - if (_data.Length < 8) - throw new InvalidDataException("OneFS: file too small for LIN-tree-root header."); - - if (_data.AsSpan(0, 5).SequenceEqual(LongTag)) { - this.Tag = "OneFS"; - this.TrailingWord = _data.Length >= 12 - ? BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(8, 4)) : 0; - } else if (_data.AsSpan(0, 4).SequenceEqual(ShortTag)) { - this.Tag = "ONEF"; - this.TrailingWord = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(4, 4)); - } else { - throw new InvalidDataException("OneFS: missing 'OneFS' / 'ONEF' tag at offset 0."); - } - - this.ValidHeader = true; - - var meta = BuildMetadata(); - _entries.Add(new OneFsEntry { Name = "metadata.ini", Size = meta.Length, IsDirectory = false, Offset = 0, Data = meta }); - _entries.Add(new OneFsEntry { Name = "onefs-volume.bin", Size = _data.Length, IsDirectory = false, Offset = 0, Data = _data }); + private void BuildEntries() { + var metadata = this.BuildMetadata(); + this._entries.Add(new OneFsEntry { + Name = MetadataEntryName, + Size = metadata.Length, + IsDirectory = false, + Offset = 0, + Data = metadata, + }); + this._entries.Add(new OneFsEntry { + Name = RawImageEntryName, + Size = this.ImageSize, + IsDirectory = false, + Offset = 0, + Data = [], + }); } private byte[] BuildMetadata() { - var bldr = new StringBuilder(); - bldr.Append("parse_status=detection-only\n"); - bldr.Append("stage=0\n"); - bldr.Append("format=Dell EMC Isilon OneFS LIN-tree root\n"); - bldr.Append(CultureInfo.InvariantCulture, $"magic_tag={this.Tag}\n"); - bldr.Append("magic_offset=0\n"); - bldr.Append(CultureInfo.InvariantCulture, $"trailing_word=0x{this.TrailingWord:X8}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"image_size={_data.Length}\n"); - bldr.Append("ro_promotion=blocked\n"); - bldr.Append("ro_promotion_reason_1=FEC-striped across nodes (N+M:B protection groups); single-image carries one stripe only\n"); - bldr.Append("ro_promotion_reason_2=LIN tree is cluster-wide, no per-image inode-to-block mapping\n"); - bldr.Append("ro_promotion_reason_3=on-disk format is proprietary; Dell EMC has not published a spec\n"); - bldr.Append("ro_promotion_reason_4=FreeBSD-kernel ancestry does NOT imply UFS on-disk compatibility (no UFS1 superblock at offset 8192)\n"); - bldr.Append("note=Stage 0 — detection only. OneFS is a clustered scale-out NAS; "); - bldr.Append("LIN tree is cluster-wide, no single-image content surface.\n"); - return Encoding.UTF8.GetBytes(bldr.ToString()); + var builder = new StringBuilder(); + builder.Append("parse_status=opaque-single-image\n"); + builder.Append("stage=0\n"); + builder.Append("format=Dell PowerScale / Isilon OneFS\n"); + builder.Append("detection=extension-or-explicit-selection\n"); + builder.Append("authoritative_raw_magic=not_published\n"); + builder.Append("physical_block_size=").Append(PhysicalBlockSize.ToString(CultureInfo.InvariantCulture)).Append('\n'); + builder.Append("image_size=").Append(this.ImageSize.ToString(CultureInfo.InvariantCulture)).Append('\n'); + builder.Append("rw_promotion=blocked\n"); + builder.Append("rw_promotion_reason_1=OneFS exposes one namespace across the cluster, not one self-contained namespace per drive\n"); + builder.Append("rw_promotion_reason_2=LIN B+ tree entries resolve logical inode numbers to mirrored inode addresses on node+drive+block tuples\n"); + builder.Append("rw_promotion_reason_3=IFM metatrees resolve logical file blocks to protection groups distributed across cluster nodes and drives\n"); + builder.Append("rw_promotion_reason_4=no published raw-media serialization/update specification or offline single-drive checker was found\n"); + builder.Append("maintenance=blocked\n"); + builder.Append("maintenance_reason=free/live allocation, relocation metadata, protection-group membership and transaction rules cannot be proven from one opaque image\n"); + builder.Append("ufs_note=Dell describes OneFS as UFS-based; that architectural ancestry does not make an isolated /ifs drive image a generic standalone UFS volume\n"); + builder.Append("note=The raw image is exposed byte-for-byte for inspection only; no undocumented bytes are interpreted or rewritten.\n"); + return Encoding.UTF8.GetBytes(builder.ToString()); } /// - /// Decodes the supplied input. + /// Opens an entry as a bounded read-only stream. Opening the raw image resets + /// the source to byte zero but never copies the payload into managed memory. /// - public byte[] Extract(OneFsEntry entry) { + public Stream OpenEntry(OneFsEntry entry) { ArgumentNullException.ThrowIfNull(entry); - return entry.Data; + + if (string.Equals(entry.Name, MetadataEntryName, StringComparison.Ordinal)) + return new BoundedEntryStream(new MemoryStream(entry.Data, writable: false), entry.Data.LongLength, leaveOpen: false); + + if (!string.Equals(entry.Name, RawImageEntryName, StringComparison.Ordinal)) + throw new FileNotFoundException($"OneFS entry not found: {entry.Name}", entry.Name); + + this._stream.Position = 0; + return new BoundedEntryStream(this._stream, this.ImageSize, leaveOpen: true); } /// - /// Releases resources held by this instance. + /// Materializes an entry in memory. Prefer for the raw + /// image so large media remains streaming. /// - public void Dispose() { } + public byte[] Extract(OneFsEntry entry) { + ArgumentNullException.ThrowIfNull(entry); + if (entry.Size > Array.MaxLength) + throw new IOException($"OneFS entry '{entry.Name}' is too large to materialize; use OpenEntry for streaming access."); + + using var source = this.OpenEntry(entry); + using var target = new MemoryStream(entry.Size > 0 ? checked((int)entry.Size) : 0); + source.CopyTo(target); + return target.ToArray(); + } + + /// Releases the source stream when ownership was requested. + public void Dispose() { + if (!this._leaveOpen) + this._stream.Dispose(); + } } From a959eedbf6846fff6e3f431e18d005513c5e701d Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:03:19 +0200 Subject: [PATCH 044/165] + cover TI-TXT mutation and canonical rebuild --- .../FirmwareHex/TiTxtMaintenanceTests.cs | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 Compression.Tests/FirmwareHex/TiTxtMaintenanceTests.cs diff --git a/Compression.Tests/FirmwareHex/TiTxtMaintenanceTests.cs b/Compression.Tests/FirmwareHex/TiTxtMaintenanceTests.cs new file mode 100644 index 000000000..b2c6bb8e0 --- /dev/null +++ b/Compression.Tests/FirmwareHex/TiTxtMaintenanceTests.cs @@ -0,0 +1,121 @@ +using System.Text; +using Compression.Registry; +using FileFormat.FirmwareHex; + +namespace Compression.Tests.FirmwareHex; + +[TestFixture] +public sealed class TiTxtMaintenanceTests { + + [Test] + public void Parser_RejectsOverlappingSections() { + const string text = "@1000\n01 02 03 04\n@1002\nAA BB\nq\n"; + Assert.That(() => TiTxtReader.Read(text), Throws.InstanceOf()); + } + + [Test] + public void Parser_RejectsTrailingContentAfterTerminator() { + const string text = "@1000\n01 02\nq\n@2000\n03 04\n"; + Assert.That(() => TiTxtReader.Read(text), Throws.InstanceOf()); + } + + [Test] + public void Defragment_CanonicalisesAdjacentSectionsWithoutFillingHoles() { + const string text = "@1010\n05 06\n@1004\n03 04\n@1000\n01 02 07 08\nq\n"; + using var image = StreamOf(text); + var descriptor = new TiTxtFormatDescriptor(); + + descriptor.Defragment(image); + + var rewritten = Encoding.ASCII.GetString(image.ToArray()); + var parsed = TiTxtReader.Read(rewritten); + Assert.Multiple(() => { + Assert.That(parsed.Segments, Has.Count.EqualTo(2)); + Assert.That(parsed.Segments[0].Address, Is.EqualTo(0x1000u)); + Assert.That(parsed.Segments[0].Data, Is.EqualTo(new byte[] { 1, 2, 7, 8, 3, 4 }).AsCollection); + Assert.That(parsed.Segments[1].Address, Is.EqualTo(0x1010u)); + Assert.That(parsed.Segments[1].Data, Is.EqualTo(new byte[] { 5, 6 }).AsCollection); + Assert.That(rewritten, Does.Not.Contain("@1004")); + Assert.That(rewritten, Does.Contain("@1010")); + }); + } + + [Test] + public void Add_ReplacesPayloadAtExistingBaseAddress() { + using var image = StreamOf("@C000\n01 02 03\nq\n"); + var descriptor = new TiTxtFormatDescriptor(); + + descriptor.Add(image, [ArchiveInputInfo.InMemory("replacement.bin", new byte[] { 0xDE, 0xAD, 0xBE, 0xEF })]); + + var parsed = Parse(image); + Assert.Multiple(() => { + Assert.That(parsed.Segments, Has.Count.EqualTo(1)); + Assert.That(parsed.Segments[0].Address, Is.EqualTo(0xC000u)); + Assert.That(parsed.Segments[0].Data, Is.EqualTo(new byte[] { 0xDE, 0xAD, 0xBE, 0xEF }).AsCollection); + }); + } + + [Test] + public void Add_MetadataOnlyShiftPreservesSparseGaps() { + using var image = StreamOf("@1000\n01 02\n@1010\n03 04\nq\n"); + var descriptor = new TiTxtFormatDescriptor(); + var metadata = Encoding.UTF8.GetBytes("[firmware_hex]\nbase_address = 0x00002000\n"); + + descriptor.Add(image, [ArchiveInputInfo.InMemory(FirmwareHexWriter.MetadataName, metadata)]); + + var parsed = Parse(image); + Assert.Multiple(() => { + Assert.That(parsed.Segments, Has.Count.EqualTo(2)); + Assert.That(parsed.Segments[0].Address, Is.EqualTo(0x2000u)); + Assert.That(parsed.Segments[1].Address, Is.EqualTo(0x2010u)); + Assert.That(parsed.GapCount, Is.EqualTo(1)); + Assert.That(parsed.TotalDataBytes, Is.EqualTo(4)); + }); + } + + [Test] + public void Remove_FirmwareLeavesValidEmptyImage() { + using var image = StreamOf("@1000\n01 02\nq\n"); + var descriptor = new TiTxtFormatDescriptor(); + + descriptor.Remove(image, [FirmwareHexWriter.PayloadName]); + + Assert.Multiple(() => { + Assert.That(Encoding.ASCII.GetString(image.ToArray()), Is.EqualTo("q\n")); + Assert.That(Parse(image).Segments, Is.Empty); + }); + } + + [Test] + public void Purge_RemovesLivePayloadAndLeavesValidEmptyImage() { + using var image = StreamOf("@1000\n01 02\n@2000\n03 04\nq\n"); + var descriptor = new TiTxtFormatDescriptor(); + + descriptor.Purge(image); + + Assert.Multiple(() => { + Assert.That(Encoding.ASCII.GetString(image.ToArray()), Is.EqualTo("q\n")); + Assert.That(Parse(image).Segments, Is.Empty); + }); + } + + [Test] + public void Create_RejectsMultiplePayloads() { + var descriptor = new TiTxtFormatDescriptor(); + using var output = new MemoryStream(); + var inputs = new[] { + ArchiveInputInfo.InMemory("a.bin", new byte[] { 1 }), + ArchiveInputInfo.InMemory("b.bin", new byte[] { 2 }), + }; + + Assert.That(() => descriptor.Create(output, inputs, new FormatCreateOptions()), + Throws.InstanceOf()); + } + + private static MemoryStream StreamOf(string text) => new(Encoding.ASCII.GetBytes(text)); + + private static FirmwareImage Parse(MemoryStream stream) { + stream.Position = 0; + return TiTxtReader.Read(Encoding.ASCII.GetString(stream.ToArray())); + } +} From dd97b4c713ab83562011396ef32aceed71f5a646 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:03:31 +0200 Subject: [PATCH 045/165] Cover NRG chunk interop and maintenance --- Compression.Tests/Nrg/NrgTests.cs | 326 ++++++++++++++++++------------ 1 file changed, 193 insertions(+), 133 deletions(-) diff --git a/Compression.Tests/Nrg/NrgTests.cs b/Compression.Tests/Nrg/NrgTests.cs index f15422dc1..673cf3863 100644 --- a/Compression.Tests/Nrg/NrgTests.cs +++ b/Compression.Tests/Nrg/NrgTests.cs @@ -1,164 +1,224 @@ +using System.Buffers.Binary; +using Compression.Registry; +using FileFormat.Nrg; + namespace Compression.Tests.Nrg; [TestFixture] public class NrgTests { + private static byte[] CreateNrg(params (string Name, byte[] Data)[] files) { + var descriptor = new NrgFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create( + image, + files.Select(static file => ArchiveInputInfo.InMemory(file.Name, file.Data)).ToArray(), + new FormatCreateOptions()); + return image.ToArray(); + } + + private static byte[] BuildOffsetTrackNrg(string fileName, byte[] data, int prefixLength = 4096) { + var canonical = CreateNrg((fileName, data)); + var canonicalTrailer = checked((int)BinaryPrimitives.ReadUInt64BigEndian(canonical.AsSpan(canonical.Length - 8))); + var iso = canonical.AsSpan(0, canonicalTrailer).ToArray(); + + using var output = new MemoryStream(); + output.Write(new byte[prefixLength]); + output.Write(iso); + var trailerOffset = checked((ulong)output.Position); + + Span etn2 = stackalloc byte[32]; + BinaryPrimitives.WriteUInt64BigEndian(etn2, checked((ulong)prefixLength)); + BinaryPrimitives.WriteUInt64BigEndian(etn2[8..], checked((ulong)iso.LongLength)); + etn2[19] = 0x00; + WriteChunk(output, "ETN2"u8, etn2); + + Span sinf = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(sinf, 1); + WriteChunk(output, "SINF"u8, sinf); + + Span mtyp = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(mtyp, 0x00000400); + WriteChunk(output, "MTYP"u8, mtyp); + WriteChunk(output, "END!"u8, ReadOnlySpan.Empty); + + Span footer = stackalloc byte[12]; + "NER5"u8.CopyTo(footer); + BinaryPrimitives.WriteUInt64BigEndian(footer[4..], trailerOffset); + output.Write(footer); + return output.ToArray(); + } - // Builds a minimal NRG v2 image: flat ISO sectors followed by the NER5 footer. - // The footer is 12 bytes: "NER5" + uint64 BE offset to chunk table (appended after data). - private static byte[] BuildNrg(string fileName, byte[] fileData) { - const int sectorSize = 2048; - const int pvdLba = 16; - const int rootDirLba = 18; - const int fileLba = 19; - - var totalSectors = fileLba + 2; - var isoBuf = new byte[totalSectors * sectorSize]; - - // File data - fileData.AsSpan().CopyTo(isoBuf.AsSpan(fileLba * sectorSize)); - - // Root directory - var dirPos = rootDirLba * sectorSize; - var dot = new byte[34]; dot[0] = 34; dot[2] = rootDirLba; dot[25] = 0x02; dot[32] = 1; dot[33] = 0x00; - dot.AsSpan().CopyTo(isoBuf.AsSpan(dirPos)); dirPos += 34; - var dotdot = new byte[34]; dotdot[0] = 34; dotdot[2] = rootDirLba; dotdot[25] = 0x02; dotdot[32] = 1; dotdot[33] = 0x01; - dotdot.AsSpan().CopyTo(isoBuf.AsSpan(dirPos)); dirPos += 34; - - var isoName = fileName.ToUpperInvariant() + ";1"; - var idLen = (byte)isoName.Length; - var recLen = (byte)(33 + idLen + ((33 + idLen) % 2)); - var rec = new byte[recLen]; - rec[0] = recLen; rec[2] = (byte)fileLba; rec[6] = (byte)fileLba; - var sz = (uint)fileData.Length; BitConverter.GetBytes(sz).CopyTo(rec, 10); // LE at offset 10 - rec[32] = idLen; System.Text.Encoding.ASCII.GetBytes(isoName).CopyTo(rec, 33); - rec.AsSpan().CopyTo(isoBuf.AsSpan(dirPos)); - dirPos += recLen; - - var pvd = new byte[sectorSize]; - pvd[0] = 1; System.Text.Encoding.ASCII.GetBytes("CD001").CopyTo(pvd, 1); pvd[6] = 1; - pvd[156] = 34; pvd[158] = rootDirLba; pvd[162] = rootDirLba; - pvd[166] = (byte)(dirPos - rootDirLba * sectorSize); pvd[170] = pvd[166]; - pvd[156 + 25] = 0x02; pvd[156 + 32] = 1; - pvd.AsSpan().CopyTo(isoBuf.AsSpan(pvdLba * sectorSize)); - - // NRG v2 footer: "NER5" + uint64 BE (chunk table offset = end of data area) - var chunkOffset = (ulong)isoBuf.Length; - var footer = new byte[12]; - footer[0] = (byte)'N'; footer[1] = (byte)'E'; footer[2] = (byte)'R'; footer[3] = (byte)'5'; - footer[4] = (byte)(chunkOffset >> 56); - footer[5] = (byte)(chunkOffset >> 48); - footer[6] = (byte)(chunkOffset >> 40); - footer[7] = (byte)(chunkOffset >> 32); - footer[8] = (byte)(chunkOffset >> 24); - footer[9] = (byte)(chunkOffset >> 16); - footer[10] = (byte)(chunkOffset >> 8); - footer[11] = (byte)chunkOffset; - - var result = new byte[isoBuf.Length + footer.Length]; - isoBuf.AsSpan().CopyTo(result); - footer.AsSpan().CopyTo(result.AsSpan(isoBuf.Length)); - return result; + private static void WriteChunk(Stream output, ReadOnlySpan id, ReadOnlySpan payload) { + Span header = stackalloc byte[8]; + id.CopyTo(header); + BinaryPrimitives.WriteUInt32BigEndian(header[4..], checked((uint)payload.Length)); + output.Write(header); + output.Write(payload); + } + + private static byte[] ReadFile(byte[] image, string leaf) { + using var stream = new MemoryStream(image, writable: false); + using var reader = new NrgReader(stream); + var entry = reader.Entries.Single(e => !e.IsDirectory && e.Name.Equals(leaf, StringComparison.OrdinalIgnoreCase)); + return reader.Extract(entry); } [Test, Category("HappyPath")] public void Read_Nrg_DetectsVersion2Footer() { - var data = "NRG content"u8.ToArray(); - var nrg = BuildNrg("readme.txt", data); - using var ms = new MemoryStream(nrg); + using var stream = new MemoryStream(CreateNrg(("README.TXT", "NRG content"u8.ToArray()))); + using var reader = new NrgReader(stream); + Assert.That(reader.Version, Is.EqualTo(2)); + } - var r = new FileFormat.Nrg.NrgReader(ms); - Assert.That(r.Version, Is.EqualTo(2)); + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Create_WritesRealV2ChunkChain() { + var image = CreateNrg(("DATA.BIN", "nero-payload"u8.ToArray())); + Assert.That(image.AsSpan(image.Length - 12, 4).ToArray(), Is.EqualTo("NER5"u8.ToArray())); + + var trailerOffset = checked((int)BinaryPrimitives.ReadUInt64BigEndian(image.AsSpan(image.Length - 8))); + Assert.That(trailerOffset, Is.LessThan(image.Length - 12)); + + var position = trailerOffset; + AssertChunk(image, ref position, "ETN2", 32, payload => { + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(payload), Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(payload[8..]), Is.EqualTo((ulong)trailerOffset)); + Assert.That(payload[19], Is.Zero, "cooked Mode 1"); + }); + AssertChunk(image, ref position, "SINF", 4, + payload => Assert.That(BinaryPrimitives.ReadUInt32BigEndian(payload), Is.EqualTo(1))); + AssertChunk(image, ref position, "MTYP", 4, + payload => Assert.That(BinaryPrimitives.ReadUInt32BigEndian(payload), Is.EqualTo(0x00000400))); + AssertChunk(image, ref position, "END!", 0, null); + Assert.That(position, Is.EqualTo(image.Length - 12)); } - [Test, Category("HappyPath")] - public void Read_Nrg_ListsFile() { - var data = "Nero disc data"u8.ToArray(); - var nrg = BuildNrg("file.dat", data); - using var ms = new MemoryStream(nrg); + private static void AssertChunk(byte[] image, ref int position, string id, int expectedLength, Action>? inspect) { + Assert.That(System.Text.Encoding.ASCII.GetString(image, position, 4), Is.EqualTo(id)); + var length = checked((int)BinaryPrimitives.ReadUInt32BigEndian(image.AsSpan(position + 4, 4))); + Assert.That(length, Is.EqualTo(expectedLength)); + var payload = image.AsSpan(position + 8, length); + inspect?.Invoke(payload); + position += 8 + length; + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Read_UsesEtn2TrackOffset_InsteadOfAssumingOffsetZero() { + var payload = "track-offset"u8.ToArray(); + var image = BuildOffsetTrackNrg("OFFSET.BIN", payload); - var r = new FileFormat.Nrg.NrgReader(ms); - Assert.That(r.Entries, Has.Count.GreaterThan(0)); - var file = r.Entries.FirstOrDefault(e => !e.IsDirectory); - Assert.That(file, Is.Not.Null); - Assert.That(file!.Size, Is.EqualTo(data.Length)); + Assert.That(ReadFile(image, "OFFSET.BIN"), Is.EqualTo(payload)); } - [Test, Category("HappyPath")] - public void Read_Nrg_ExtractReturnsData() { - var data = new byte[128]; - Random.Shared.NextBytes(data); - var nrg = BuildNrg("sample.bin", data); - using var ms = new MemoryStream(nrg); + [Test, Category("Regression")] + public void Read_TruncatedChunkLength_FailsClosedAndFallsBackToLegacyTrackProbe() { + var canonical = CreateNrg(("SAFE.BIN", "safe"u8.ToArray())); + var trailer = checked((int)BinaryPrimitives.ReadUInt64BigEndian(canonical.AsSpan(canonical.Length - 8))); + var iso = canonical.AsSpan(0, trailer).ToArray(); + + using var output = new MemoryStream(); + output.Write(iso); + var malformedTrailer = checked((ulong)output.Position); + output.Write("ETN2"u8); + Span absurdLength = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(absurdLength, uint.MaxValue); + output.Write(absurdLength); + Span footer = stackalloc byte[12]; + "NER5"u8.CopyTo(footer); + BinaryPrimitives.WriteUInt64BigEndian(footer[4..], malformedTrailer); + output.Write(footer); + + Assert.That(ReadFile(output.ToArray(), "SAFE.BIN"), Is.EqualTo("safe"u8.ToArray())); + } - var r = new FileFormat.Nrg.NrgReader(ms); - var file = r.Entries.FirstOrDefault(e => !e.IsDirectory); - Assert.That(file, Is.Not.Null); + [Test, Category("RoundTrip")] + public void Create_PreservesDirectoryPaths() { + var image = CreateNrg(("DIR/SUB/FILE.BIN", "nested"u8.ToArray())); + using var stream = new MemoryStream(image); + using var reader = new NrgReader(stream); - var extracted = r.Extract(file!); - Assert.That(extracted[..data.Length], Is.EqualTo(data)); + Assert.That(reader.Entries.Any(e => !e.IsDirectory && e.FullPath.Equals("DIR/SUB/FILE.BIN", StringComparison.OrdinalIgnoreCase)), Is.True); + Assert.That(ReadFile(image, "FILE.BIN"), Is.EqualTo("nested"u8.ToArray())); } - [Test, Category("HappyPath")] - public void Read_NoFooter_VersionIsZero() { - // Stream with no NRG footer — version should be 0 - using var ms = new MemoryStream(new byte[2352 * 32]); - var r = new FileFormat.Nrg.NrgReader(ms); - Assert.That(r.Version, Is.EqualTo(0)); + [Test, Category("RoundTrip")] + public void Modify_AddReplaceRemove_RoundTripsNamedIsoEntries() { + var descriptor = new NrgFormatDescriptor(); + using var image = new MemoryStream(); + image.Write(CreateNrg(("SEED.BIN", "seed"u8.ToArray()))); + image.Position = 0; + + ((IArchiveModifiable)descriptor).Add(image, [ArchiveInputInfo.InMemory("PROBE.BIN", "probe"u8.ToArray())]); + Assert.That(ReadFile(image.ToArray(), "SEED.BIN"), Is.EqualTo("seed"u8.ToArray())); + Assert.That(ReadFile(image.ToArray(), "PROBE.BIN"), Is.EqualTo("probe"u8.ToArray())); + + ((IArchiveModifiable)descriptor).Add(image, [ArchiveInputInfo.InMemory("PROBE.BIN", "replaced"u8.ToArray())]); + Assert.That(ReadFile(image.ToArray(), "PROBE.BIN"), Is.EqualTo("replaced"u8.ToArray())); + + ((IArchiveModifiable)descriptor).Remove(image, ["PROBE.BIN"]); + using var after = new MemoryStream(image.ToArray()); + using var reader = new NrgReader(after); + Assert.That(reader.Entries.Any(e => !e.IsDirectory && e.Name.Equals("PROBE.BIN", StringComparison.OrdinalIgnoreCase)), Is.False); + Assert.That(ReadFile(image.ToArray(), "SEED.BIN"), Is.EqualTo("seed"u8.ToArray())); } - [Test, Category("HappyPath")] - public void NrgEntry_Properties() { - var entry = new FileFormat.Nrg.NrgEntry { - Name = "AUTORUN.INF", - FullPath = "AUTORUN.INF", - IsDirectory = false, - Size = 128, - StartLba = 20, - }; - Assert.That(entry.Name, Is.EqualTo("AUTORUN.INF")); - Assert.That(entry.FullPath, Is.EqualTo("AUTORUN.INF")); - Assert.That(entry.IsDirectory, Is.False); - Assert.That(entry.Size, Is.EqualTo(128)); - Assert.That(entry.StartLba, Is.EqualTo(20)); + [Test, Category("RoundTrip")] + public void Maintenance_RebuildsOffsetImage_WithoutChangingPayload() { + var payload = Enumerable.Range(0, 1000).Select(static i => (byte)(i * 31)).ToArray(); + var source = BuildOffsetTrackNrg("KEEP.BIN", payload, prefixLength: 8192); + var descriptor = new NrgFormatDescriptor(); + + using var defrag = new MemoryStream(); + defrag.Write(source); + defrag.Position = 0; + ((IArchiveDefragmentable)descriptor).Defragment(defrag); + Assert.That(ReadFile(defrag.ToArray(), "KEEP.BIN"), Is.EqualTo(payload)); + + using var input = new MemoryStream(source); + using var shrunk = new MemoryStream(); + ((IArchiveShrinkable)descriptor).Shrink(input, shrunk); + Assert.That(shrunk.Length, Is.LessThan(source.Length)); + Assert.That(ReadFile(shrunk.ToArray(), "KEEP.BIN"), Is.EqualTo(payload)); + } + + [Test, Category("RoundTrip")] + public void Purge_LeavesValidEmptyNrg() { + var descriptor = new NrgFormatDescriptor(); + using var image = new MemoryStream(); + image.Write(CreateNrg(("DELETE.ME", "gone"u8.ToArray()))); + image.Position = 0; + + ((IArchivePurgeable)descriptor).Purge(image); + + image.Position = 0; + using var reader = new NrgReader(image, leaveOpen: true); + Assert.That(reader.Version, Is.EqualTo(2)); + Assert.That(reader.Entries.Where(static e => !e.IsDirectory), Is.Empty); } [Test, Category("HappyPath")] - public void Detect_ByExtension() { - var format = Compression.Lib.FormatDetector.DetectByExtension("disc.nrg"); - Assert.That(format, Is.EqualTo(Compression.Lib.FormatDetector.Format.Nrg)); + public void Descriptor_AdvertisesOnlySupportedMaintenanceVerbs() { + var descriptor = new NrgFormatDescriptor(); + Assert.Multiple(() => { + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + }); } [Test, Category("HappyPath")] - public void Descriptor_ReportsWormCapability() { - var d = new FileFormat.Nrg.NrgFormatDescriptor(); - Assert.That(d.Capabilities.HasFlag(Compression.Registry.FormatCapabilities.CanCreate), Is.True); + public void Read_NoFooter_VersionIsZero() { + using var stream = new MemoryStream(new byte[2352 * 32]); + using var reader = new NrgReader(stream); + Assert.That(reader.Version, Is.Zero); } - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Create_HasNer5Footer_AndRoundTrips() { - var payload = "nero-payload"u8.ToArray(); - var tmp = Path.GetTempFileName(); - try { - File.WriteAllBytes(tmp, payload); - var d = new FileFormat.Nrg.NrgFormatDescriptor(); - using var ms = new MemoryStream(); - ((Compression.Registry.IArchiveCreatable)d).Create( - ms, - [new Compression.Registry.ArchiveInputInfo(tmp, "data.bin", false)], - new Compression.Registry.FormatCreateOptions()); - - // Verify the NER5 footer was emitted (last 12 bytes: "NER5" + uint64 BE). - var bytes = ms.ToArray(); - Assert.That(bytes[^12..^8], Is.EqualTo(new byte[] { (byte)'N', (byte)'E', (byte)'R', (byte)'5' })); - - ms.Position = 0; - var r = new FileFormat.Nrg.NrgReader(ms); - Assert.That(r.Version, Is.EqualTo(2)); - var fileEntry = r.Entries.FirstOrDefault(e => !e.IsDirectory && e.Name.StartsWith("DATA")); - Assert.That(fileEntry, Is.Not.Null); - Assert.That(r.Extract(fileEntry!)[..payload.Length], Is.EqualTo(payload)); - } finally { - File.Delete(tmp); - } + [Test, Category("HappyPath")] + public void Detect_ByExtension() { + var format = Compression.Lib.FormatDetector.DetectByExtension("disc.nrg"); + Assert.That(format, Is.EqualTo(Compression.Lib.FormatDetector.Format.Nrg)); } } From 200cbdda1f22feeb864887bb0e912c34cf589e55 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:03:33 +0200 Subject: [PATCH 046/165] Tighten Intel HEX test nullability --- Compression.Tests/FirmwareHex/IntelHexRwTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Compression.Tests/FirmwareHex/IntelHexRwTests.cs b/Compression.Tests/FirmwareHex/IntelHexRwTests.cs index e42a81b25..70fb61593 100644 --- a/Compression.Tests/FirmwareHex/IntelHexRwTests.cs +++ b/Compression.Tests/FirmwareHex/IntelHexRwTests.cs @@ -132,7 +132,7 @@ private static Dictionary Extract(IntelHexFormatDescriptor descr using var input = new MemoryStream(image, writable: false); descriptor.Extract(input, directory, null, null); return Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories) - .ToDictionary(path => Path.GetFileName(path), File.ReadAllBytes, StringComparer.OrdinalIgnoreCase); + .ToDictionary(path => Path.GetFileName(path)!, File.ReadAllBytes, StringComparer.OrdinalIgnoreCase); } finally { try { Directory.Delete(directory, recursive: true); } catch { /* best effort */ } } From e80380a20265b8c01a87e4b2f90445ff63aebadf Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:03:34 +0200 Subject: [PATCH 047/165] fix(onefs): stop advertising unverified raw magic --- .../FileSystem.OneFs/OneFsFormatDescriptor.cs | 202 ++++++++---------- 1 file changed, 84 insertions(+), 118 deletions(-) diff --git a/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs b/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs index e9f8a976a..5ab7ff6f5 100644 --- a/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs +++ b/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs @@ -1,160 +1,126 @@ #pragma warning disable CS1591 using Compression.Registry; -using Compression.Registry.Streaming; using static Compression.Registry.FormatHelpers; namespace FileSystem.OneFs; /// -/// Stage 0 detection-only descriptor for Dell EMC Isilon OneFS LIN-tree -/// root images. Surfaces only a synthetic metadata.ini and the raw -/// image bytes; no real file-walk is attempted. -/// +/// Conservative inspection descriptor for Dell PowerScale / Isilon OneFS media. +/// +/// /// -/// Why R/O promotion is impossible (per CONTRIBUTING.md promotion gates): +/// OneFS is a distributed filesystem. Dell documents a global namespace, a LIN +/// B+ tree whose values are mirrored inode addresses on node/drive/block tuples, +/// and IFM metatrees that map logical file blocks into protection groups spread +/// through the cluster. A single raw drive therefore does not carry enough state +/// to reconstruct arbitrary files or safely rewrite their allocation metadata. +/// +/// +/// No Dell publication found during the implementation review defines the +/// historical "OneFS" or "ONEF" literals as fixed offset-zero raw +/// media signatures. Advertising those bytes caused unrelated files beginning +/// with either string to be identified as OneFS. Detection is consequently by +/// explicit selection or the unambiguous .onefs extension only. /// -/// -/// -/// -/// No single-image content surface. OneFS is a clustered scale-out NAS -/// — every file is split into "protection groups" striped across drives and -/// nodes with FEC (Forward Error Correction, N+M:B layout, e.g. N+2:1). A -/// single drive/node image carries only one stripe; the file data cannot be -/// reconstructed without the peer nodes. A read-only reader from one image -/// can never return correct file bytes. -/// -/// -/// -/// -/// LIN tree is cluster-wide. The Logical Inode Number tree (the OneFS -/// metadata index) lives across nodes, not in a single superblock. There is -/// no per-image inode-to-block mapping to walk. -/// -/// -/// -/// -/// Proprietary on-disk format, no public specification. Dell EMC has -/// never published the OneFS on-disk format. No open-source reverse-engineered -/// reader exists. Without a spec we cannot honour the CONTRIBUTING rule -/// "never advertise capabilities you cannot prove against a real spec". -/// -/// -/// -/// -/// FreeBSD/UFS ancestry does NOT give us a UFS reader fallback. OneFS -/// runs on a FreeBSD-derived kernel, but the filesystem layer is entirely -/// proprietary — it is NOT FFS/UFS at the on-disk level. UFS1 places its -/// superblock magic 0x00011954 at offset 8192; OneFS images have the -/// ASCII "OneFS" tag at offset 0 and no UFS superblock. Routing OneFS -/// images through UfsReader would fail the magic check and (if forced) -/// return arbitrary bytes — the textbook mutual-compensation trap. -/// -/// -/// -/// /// -/// Conclusion: Stage-0 detection only. Surface the magic, raw bytes, -/// and a metadata.ini documenting the limitation. R/O promotion is -/// blocked on (a) Dell EMC publishing the spec and (b) a multi-node ingest -/// path — neither is in reach. +/// R/W and the destructive maintenance verbs remain deliberately unavailable. +/// Implementing add/remove, purge, wipe, relocation/defrag, shrink or relayout +/// without the raw allocation, protection and transaction formats would corrupt +/// media while satisfying only this library's own synthetic tests. The reader +/// instead exposes metadata describing the known architecture plus the original +/// image as a byte-exact bounded stream. /// -/// -/// References: +/// +/// Clean-room references: /// -/// Dell EMC "PowerScale OneFS Technical Overview" whitepaper — high-level architecture only; no on-disk spec is published -/// https://en.wikipedia.org/wiki/OneFS_distributed_file_system — Wikipedia article +/// https://infohub.delltechnologies.com/en-us/p/onefs-metadata/ +/// — LIN tree, inode mirrors, IFM/DFM B+ trees and protection metadata. +/// https://infohub.delltechnologies.com/en-nz/l/dell-powerscale-onefs-technical-overview/file-system-structure/1/ +/// — distributed UFS-based filesystem and global namespace. +/// Dell PowerScale OneFS Technical Specifications Guide — +/// fixed 8 KiB filesystem block size. /// -/// +/// No external implementation code is copied or translated. +/// +/// public sealed class OneFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { - /// - /// Gets the id. - /// + /// Gets the registry id. public string Id => "OneFs"; - /// - /// Gets the display name. - /// + + /// Gets the display name. public string DisplayName => "Dell EMC Isilon OneFS"; - /// - /// Gets the category. - /// + + /// Gets the format category. public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// + + /// Gets the conservative read-only capabilities. public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest; - /// - /// Gets the default extension. - /// + + /// Gets the conventional extension for explicitly supplied raw media. public string DefaultExtension => ".onefs"; - /// - /// Gets the extensions. - /// + + /// Gets the recognized extensions. public IReadOnlyList Extensions => [".onefs"]; - /// - /// Gets the compound extensions. - /// + + /// Gets compound extensions. public IReadOnlyList CompoundExtensions => []; + /// - /// Gets the magic signatures. - /// - public IReadOnlyList MagicSignatures => [ - // ASCII "OneFS" (5 bytes) at offset 0 — long tag form. - new("OneFS"u8.ToArray(), Offset: 0, Confidence: 0.90), - // ASCII "ONEF" (0x4F4E4546 BE) at offset 0 — short tag form. - new("ONEF"u8.ToArray(), Offset: 0, Confidence: 0.85), - ]; - /// - /// Gets the methods. + /// Gets fixed magic signatures. Dell does not publish an authoritative + /// offset-zero raw-media signature, so OneFS intentionally has none. /// + public IReadOnlyList MagicSignatures => []; + + /// Gets the pseudo-archive storage method. public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// + + /// Gets the tar compression format id. public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// + + /// Gets the algorithm family. public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// + + /// Gets a description of the intentionally limited inspection surface. public string Description => - "Dell EMC Isilon OneFS — Stage 0, detection-only — proprietary distributed/clustered FS, " + - "no single-image content surface (file data is FEC-striped across nodes). " + - "FreeBSD-derived kernel but filesystem layer is NOT UFS-compatible (no UFS1 superblock at 8192). " + - "No public on-disk spec; R/O promotion blocked. " + - "Magic 'OneFS' / 'ONEF' at offset 0 of LIN-tree root."; + "Dell PowerScale / Isilon OneFS — opaque single-image inspection only. " + + "OneFS presents a cluster-wide namespace: LIN metadata resolves to mirrored node/drive/block addresses " + + "and file metatrees resolve data through distributed protection groups. Dell documents OneFS as UFS-based, " + + "but does not publish a standalone raw-drive layout or fixed offset-zero media magic sufficient for offline parsing. " + + "The historic 'OneFS'/'ONEF' signature claim was therefore removed. R/W, purge, wipe, defrag, shrink and relayout " + + "remain blocked until the raw allocation/protection/transaction format can be proven against a real cluster or checker."; - /// - /// Lists the entries in the supplied container. - /// + /// Lists the two conservative inspection entries without reading the image payload. public List List(Stream stream, string? password) { - var r = new OneFsReader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo( - i, e.Name, e.Size, e.Size, "Stored", e.IsDirectory, false, null)).ToList(); + using var reader = new OneFsReader(stream, leaveOpen: true); + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo( + index, entry.Name, entry.Size, entry.Size, "Stored", entry.IsDirectory, false, null)).ToList(); } - /// - /// Decodes the supplied input. - /// + /// Extracts selected inspection entries through bounded streams. public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new OneFsReader(stream); - foreach (var e in r.Entries) { - if (e.IsDirectory) continue; - if (files != null && !MatchesFilter(e.Name, files)) continue; - WriteFile(outputDir, e.Name, r.Extract(e)); + using var reader = new OneFsReader(stream, leaveOpen: true); + foreach (var entry in reader.Entries) { + if (entry.IsDirectory) + continue; + if (files is { Length: > 0 } && !MatchesFilter(entry.Name, files)) + continue; + + using var source = reader.OpenEntry(entry); + using var target = CreateEntryFile(outputDir, entry.Name); + source.CopyTo(target); } } Stream IArchiveFormatOperations.OpenEntry(Stream archive, string entryName, string? password) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(entryName); - var r = new OneFsReader(archive); - var entry = r.Entries.FirstOrDefault(e => e.Name == entryName) - ?? throw new FileNotFoundException($"OneFS entry not found: {entryName}"); - var data = r.Extract(entry); - return new BoundedEntryStream(new MemoryStream(data, writable: false), data.Length, leaveOpen: false); + + using var reader = new OneFsReader(archive, leaveOpen: true); + var entry = reader.Entries.FirstOrDefault(candidate => + string.Equals(candidate.Name, entryName, StringComparison.Ordinal)) + ?? throw new FileNotFoundException($"OneFS entry not found: {entryName}", entryName); + return reader.OpenEntry(entry); } } From aef9c3e3cfadd8a50419b2feaa20c03b517a89ca Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:03:49 +0200 Subject: [PATCH 048/165] Complete CSO ZSO read write and maintenance support --- .../FileFormat.Cso/CsoFormatDescriptor.cs | 539 ++++++++++-------- 1 file changed, 288 insertions(+), 251 deletions(-) diff --git a/FileFormats/FileFormat.Cso/CsoFormatDescriptor.cs b/FileFormats/FileFormat.Cso/CsoFormatDescriptor.cs index afe02413a..5ebb0e2eb 100644 --- a/FileFormats/FileFormat.Cso/CsoFormatDescriptor.cs +++ b/FileFormats/FileFormat.Cso/CsoFormatDescriptor.cs @@ -8,345 +8,382 @@ namespace FileFormat.Cso; /// -/// PSP CSO / ZSO compressed ISO image. Layout after the 4-byte magic (CISO for CSO, ZISO -/// for LZ4-compressed ZSO): uint32 header_size, uint64 uncompressed_size, uint32 block_size, -/// uint8 version, uint8 align, uint16 reserved, then an index table of N = uncompressed_size / -/// block_size + 1 uint32 entries (high bit = stored/uncompressed, low 31 bits = file offset). -/// -/// This descriptor surfaces each compressed block as a raw blob — it does NOT decompress -/// the blocks (consumers can further process with zlib for CSO or LZ4 for ZSO). -/// -/// References: -/// -/// https://github.com/unknownbrackets/maxcso — maxcso — maintained CSO/ZSO tool; its docs describe the CSO v1/v2 and ZSO layouts -/// The format originates in PSP homebrew (ciso); there is no official Sony documentation -/// +/// PSP CSO v1/v2 and ZSO compressed ISO image. /// +/// +/// The synthetic block entries expose logical, decompressed block contents. Replacing one therefore +/// round-trips through the same bytes a consumer of the ISO sees, while container maintenance can +/// canonicalize physical compression, index alignment and stale tail bytes independently. +/// public sealed class CsoFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, - IArchiveCreatable, IArchiveModifiable { - /// - /// Gets the id. - /// + IArchiveCreatable, IArchiveModifiable, IArchiveDefragmentable, IArchiveShrinkable, + IArchiveLayoutMap, ILayoutOptimizable, IArchivePurgeable, ISyntheticEntryNames { + + private static readonly IReadOnlySet SyntheticNames = new HashSet(StringComparer.OrdinalIgnoreCase) { + "FULL.cso", "FULL.ziso", "metadata.ini", "index.bin", + }; + public string Id => "Cso"; - /// - /// Gets the display name. - /// public string DisplayName => "PSP CSO/ZSO"; - /// - /// Gets the category. - /// public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; - /// - /// Gets the default extension. - /// public string DefaultExtension => ".cso"; - /// - /// Gets the extensions. - /// public IReadOnlyList Extensions => [".cso", ".ziso", ".zso"]; - /// - /// Gets the compound extensions. - /// public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// public IReadOnlyList MagicSignatures => [ new("CISO"u8.ToArray(), Confidence: 0.90), new("ZISO"u8.ToArray(), Confidence: 0.90), ]; - /// - /// Gets the methods. - /// public IReadOnlyList Methods => [ new("stored", "Stored"), - new("deflate", "Deflate"), - new("lz4", "LZ4"), + new("deflate", "Deflate / CSO v1"), + new("lz4", "LZ4 / ZSO"), + new("cso2", "CSO v2 (DEFLATE/LZ4)"), ]; - /// - /// Gets the tar compression format id. - /// public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// - public string Description => "PSP CSO (zlib) / ZSO (LZ4) compressed ISO image."; - - private const uint IndexUncompressedMask = 0x8000_0000u; - private const uint IndexOffsetMask = 0x7FFF_FFFFu; - - private sealed record CsoLayout( - long FullSize, - bool IsZso, - string Magic, - uint HeaderSize, - ulong UncompressedSize, - uint BlockSize, - byte Version, - byte Align, - int BlockCount, - uint[] IndexRaw - ); + public string Description => "PSP CSO v1/v2 (DEFLATE/LZ4) and ZSO (LZ4) compressed ISO image."; + public IReadOnlySet SyntheticEntryNames => SyntheticNames; - /// - /// Lists the entries in the supplied container. - /// public List List(Stream stream, string? password) { - var layout = ReadLayout(stream); + var layout = CsoImage.ReadLayout(stream); var entries = new List(4 + layout.BlockCount); - var ext = layout.IsZso ? "ziso" : "cso"; + var ext = layout.Variant == CsoVariant.Zso ? "ziso" : "cso"; entries.Add(new ArchiveEntryInfo(0, $"FULL.{ext}", layout.FullSize, layout.FullSize, "Stored", false, false, null)); entries.Add(new ArchiveEntryInfo(1, "metadata.ini", 0, 0, "Stored", false, false, null)); entries.Add(new ArchiveEntryInfo(2, "index.bin", layout.IndexRaw.Length * 4L, layout.IndexRaw.Length * 4L, "Stored", false, false, null)); entries.Add(new ArchiveEntryInfo(3, "blocks", 0, 0, "Stored", true, false, null)); - var storedMethod = layout.IsZso ? "LZ4" : "Deflate"; for (var i = 0; i < layout.BlockCount; ++i) { - var (offset, size, isUncompressed) = GetBlockSpan(layout, i); - var method = isUncompressed ? "Stored" : storedMethod; + var (_, physicalLength, encoding) = CsoImage.GetBlockSpan(layout, i); entries.Add(new ArchiveEntryInfo( Index: 4 + i, Name: $"blocks/block_{i:D5}.bin", - OriginalSize: size, - CompressedSize: size, - Method: method, + OriginalSize: CsoImage.LogicalBlockLength(layout, i), + CompressedSize: physicalLength, + Method: MethodName(encoding), IsDirectory: false, IsEncrypted: false, LastModified: null, - Kind: isUncompressed ? "stored" : "compressed")); - // offset intentionally referenced to avoid unused-local warning in contexts that strip. - _ = offset; + Kind: encoding == CsoBlockEncoding.Stored ? "stored" : "compressed")); } return entries; } - /// - /// Decodes the supplied input. - /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var layout = ReadLayout(stream); - var ext = layout.IsZso ? "ziso" : "cso"; + var layout = CsoImage.ReadLayout(stream); + var ext = layout.Variant == CsoVariant.Zso ? "ziso" : "cso"; var fullName = $"FULL.{ext}"; if (Wants(files, fullName)) { + if (layout.FullSize > int.MaxValue) + throw new InvalidDataException("CSO/ZSO whole-image synthetic entry is too large for buffered extraction."); stream.Position = 0; - var buf = new byte[layout.FullSize]; - ReadExact(stream, buf); - WriteFile(outputDir, fullName, buf); + var bytes = new byte[(int)layout.FullSize]; + CsoImage.ReadExact(stream, bytes); + WriteFile(outputDir, fullName, bytes); } if (Wants(files, "metadata.ini")) WriteFile(outputDir, "metadata.ini", Encoding.UTF8.GetBytes(BuildMetadataIni(layout))); if (Wants(files, "index.bin")) { - var idxBytes = new byte[layout.IndexRaw.Length * 4]; + var indexBytes = new byte[layout.IndexRaw.Length * sizeof(uint)]; for (var i = 0; i < layout.IndexRaw.Length; ++i) - BinaryPrimitives.WriteUInt32LittleEndian(idxBytes.AsSpan(i * 4, 4), layout.IndexRaw[i]); - WriteFile(outputDir, "index.bin", idxBytes); + BinaryPrimitives.WriteUInt32LittleEndian(indexBytes.AsSpan(i * sizeof(uint), sizeof(uint)), layout.IndexRaw[i]); + WriteFile(outputDir, "index.bin", indexBytes); } for (var i = 0; i < layout.BlockCount; ++i) { var name = $"blocks/block_{i:D5}.bin"; - if (!Wants(files, name)) continue; - var (offset, size, _) = GetBlockSpan(layout, i); - var data = ReadRange(stream, offset, size); - WriteFile(outputDir, name, data); + if (!Wants(files, name)) + continue; + var decoded = CsoImage.DecodeBlock(stream, layout, i); + var logicalLength = CsoImage.LogicalBlockLength(layout, i); + WriteFile(outputDir, name, decoded.AsSpan(0, logicalLength).ToArray()); } } - private static bool Wants(string[]? files, string name) - => files == null || files.Length == 0 || MatchesFilter(name, files); + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + ArgumentNullException.ThrowIfNull(options); - private static CsoLayout ReadLayout(Stream stream) { - if (!stream.CanSeek) - throw new InvalidDataException("CSO/ZSO descriptor requires a seekable stream."); - stream.Position = 0; - var fullSize = stream.Length; - - Span header = stackalloc byte[24]; - ReadExact(stream, header); - var magic = Encoding.ASCII.GetString(header[..4]); - var isZso = magic switch { - "CISO" => false, - "ZISO" => true, - _ => throw new InvalidDataException("Not a CSO/ZSO image (missing CISO/ZISO magic)."), - }; + var variant = VariantFromCreateOptions(options); + var blockSize = options.GetOptionInt("BlockSize", CsoWriter.DefaultBlockSize); + if (blockSize <= 0) + throw new ArgumentOutOfRangeException(nameof(options), "CSO/ZSO BlockSize must be positive."); - var headerSize = BinaryPrimitives.ReadUInt32LittleEndian(header[4..8]); - var uncompressedSize = BinaryPrimitives.ReadUInt64LittleEndian(header[8..16]); - var blockSize = BinaryPrimitives.ReadUInt32LittleEndian(header[16..20]); - var version = header[20]; - var align = header[21]; - // header[22..24] reserved. - - if (blockSize == 0) - throw new InvalidDataException("CSO/ZSO block_size is zero."); - // Block count is uncompressed_size / block_size + 1 index entries form the table, last entry - // marks end of file. Number of actual blocks is uncompressed_size / block_size (ceil). - var blockCountLong = (long)((uncompressedSize + blockSize - 1) / blockSize); - if (blockCountLong < 0 || blockCountLong > 8_000_000) - throw new InvalidDataException($"CSO/ZSO block count implausible: {blockCountLong}."); - var blockCount = (int)blockCountLong; - - var indexCount = blockCount + 1; - var indexBytes = new byte[indexCount * 4]; - // The index table immediately follows the 24-byte header (header_size is typically 0x18=24). - stream.Position = 24; - ReadExact(stream, indexBytes); - var indexRaw = new uint[indexCount]; - for (var i = 0; i < indexCount; ++i) - indexRaw[i] = BinaryPrimitives.ReadUInt32LittleEndian(indexBytes.AsSpan(i * 4, 4)); - - return new CsoLayout( - FullSize: fullSize, - IsZso: isZso, - Magic: magic, - HeaderSize: headerSize, - UncompressedSize: uncompressedSize, - BlockSize: blockSize, - Version: version, - Align: align, - BlockCount: blockCount, - IndexRaw: indexRaw); + using var payload = CsoInPlaceModifier.CreateScratchStream(); + ulong size = 0; + foreach (var input in inputs) { + if (input.IsDirectory) + continue; + var content = input.ReadContent(); + payload.Write(content); + size = checked(size + (ulong)content.Length); + } + payload.Position = 0; + CsoWriter.Write(output, payload, size, blockSize, variant); } - private static (long Offset, long Size, bool IsUncompressed) GetBlockSpan(CsoLayout layout, int blockIndex) { - var raw = layout.IndexRaw[blockIndex]; - var nextRaw = layout.IndexRaw[blockIndex + 1]; - var isUncompressed = (raw & IndexUncompressedMask) != 0; - var offset = (long)(raw & IndexOffsetMask) << layout.Align; - var nextOffset = (long)(nextRaw & IndexOffsetMask) << layout.Align; - var size = Math.Max(0, nextOffset - offset); - if (offset < 0 || offset > layout.FullSize) - throw new InvalidDataException($"CSO/ZSO block {blockIndex} offset out of range."); - if (offset + size > layout.FullSize) - size = Math.Max(0, layout.FullSize - offset); - return (offset, size, isUncompressed); + /// Replaces logical block_NNNNN.bin entries transactionally. + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + var layout = CsoImage.ReadLayout(archive); + var replacements = new Dictionary(); + foreach (var input in inputs) { + if (input.IsDirectory) + continue; + var index = ParseBlockIndex(input.ArchiveName.Replace('\\', '/')); + if (index < 0) + throw BlockNamespaceOnly(input.ArchiveName, "added"); + if (index >= layout.BlockCount) + throw new ArgumentOutOfRangeException(nameof(inputs), $"CSO/ZSO block index {index} outside [0, {layout.BlockCount})."); + var content = input.ReadContent(); + if (content.Length != layout.BlockSize) + throw new ArgumentException( + $"Replacement block {index} must be exactly block_size ({layout.BlockSize}) bytes; got {content.Length}.", + nameof(inputs)); + replacements[index] = content; + } + if (replacements.Count != 0) + CsoInPlaceModifier.WriteBlocks(archive, replacements); } - private static string BuildMetadataIni(CsoLayout layout) { - var sb = new StringBuilder(); - sb.Append("[Cso]\n"); - sb.Append(CultureInfo.InvariantCulture, $"magic={layout.Magic}\n"); - sb.Append(CultureInfo.InvariantCulture, $"is_zso={(layout.IsZso ? 1 : 0)}\n"); - sb.Append(CultureInfo.InvariantCulture, $"header_size={layout.HeaderSize}\n"); - sb.Append(CultureInfo.InvariantCulture, $"uncompressed_size={layout.UncompressedSize}\n"); - sb.Append(CultureInfo.InvariantCulture, $"block_size={layout.BlockSize}\n"); - sb.Append(CultureInfo.InvariantCulture, $"version={layout.Version}\n"); - sb.Append(CultureInfo.InvariantCulture, $"align={layout.Align}\n"); - sb.Append(CultureInfo.InvariantCulture, $"block_count={layout.BlockCount}\n"); - sb.Append(CultureInfo.InvariantCulture, $"total_size={layout.FullSize}\n"); - return sb.ToString(); + /// Clears logical blocks to zero; it does not edit the ISO 9660 directory tree inside them. + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryNames); + var layout = CsoImage.ReadLayout(archive); + var zero = new byte[checked((int)layout.BlockSize)]; + var replacements = new Dictionary(); + foreach (var name in entryNames) { + var index = ParseBlockIndex(name.Replace('\\', '/')); + if (index < 0) + throw BlockNamespaceOnly(name, "removed"); + if (index >= layout.BlockCount) + throw new ArgumentOutOfRangeException(nameof(entryNames), $"CSO/ZSO block index {index} outside [0, {layout.BlockCount})."); + replacements[index] = zero; + } + if (replacements.Count != 0) + CsoInPlaceModifier.WriteBlocks(archive, replacements); } - private static byte[] ReadRange(Stream stream, long offset, long size) { - if (size <= 0) return []; - if (size > int.MaxValue) - throw new InvalidDataException("CSO/ZSO block too large to extract."); - stream.Position = offset; - var buf = new byte[(int)size]; - ReadExact(stream, buf); - return buf; - } + // ── Maintenance ────────────────────────────────────────────────────── - private static void ReadExact(Stream stream, Span buffer) { - var read = 0; - while (read < buffer.Length) { - var n = stream.Read(buffer[read..]); - if (n <= 0) throw new EndOfStreamException("Unexpected end of CSO/ZSO stream."); - read += n; - } + /// Canonical repack: ordered blocks, align=0, no stale/orphaned body bytes. + public void Defragment(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("CSO/ZSO defragmentation requires a readable, writable, seekable stream.", nameof(archive)); + var layout = CsoImage.ReadLayout(archive); + using var staged = CsoInPlaceModifier.CreateScratchStream(); + Repack(archive, staged, layout, checked((int)layout.BlockSize)); + Commit(staged, archive, truncate: true); } - // ── IArchiveCreatable (CSO v1 only) ─────────────────────────────────── - /// - /// Emits a fresh CSO v1 stream. Inputs are concatenated in supplied order - /// to form the uncompressed payload (the caller is responsible for ensuring - /// the result is a valid PSP ISO if PSP semantics matter). + /// Rebuilds canonically and uses it only when it is smaller; otherwise copies the source through. /// - public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); ArgumentNullException.ThrowIfNull(output); - ArgumentNullException.ThrowIfNull(inputs); - using var payload = new MemoryStream(); - foreach (var input in inputs) { - if (input.IsDirectory) continue; - payload.Write(input.ReadContent()); + using var staged = CsoInPlaceModifier.CreateScratchStream(); + var useStaged = false; + try { + var layout = CsoImage.ReadLayout(input); + Repack(input, staged, layout, checked((int)layout.BlockSize)); + useStaged = staged.Length < input.Length; + } catch { + useStaged = false; } - var bytes = CsoWriter.Build(payload.ToArray()); - output.Write(bytes, 0, bytes.Length); - } - // ── IArchiveModifiable (R/W via block-NNNNN.bin synthetic entries) ──── + output.Position = 0; + output.SetLength(0); + var chosen = useStaged ? staged : input; + chosen.Position = 0; + chosen.CopyTo(output); + } /// - /// Replaces blocks named blocks/block_NNNNN.bin (5-digit zero-padded - /// index) with the supplied payloads. Each input must be exactly the - /// container's block_size bytes. Other input names are ignored. + /// Wipes dead bytes by canonicalizing into the beginning of the same-sized stream and zeroing the + /// now-unindexed tail. If the local codecs cannot reproduce the image no larger than the source, + /// no byte is touched. /// - public void Add(Stream archive, IReadOnlyList inputs) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(inputs); - foreach (var input in inputs) { - if (input.IsDirectory) continue; - var name = input.ArchiveName.Replace('\\', '/'); - var idx = ParseBlockIndex(name); - // A name this cannot place is not a name to pass over: writing nothing and - // raising nothing reports an add that did not happen. - if (idx < 0) - throw new NotSupportedException( - $"CSO: '{input.ArchiveName}' cannot be added. A compressed ISO is edited a block at a " - + "time, so an entry has to be named 'block_NNNNN.bin' for the block it replaces. " - + "Adding a file to the ISO 9660 filesystem inside it is not something this supports."); - CsoInPlaceModifier.WriteBlock(archive, idx, input.ReadContent()); + public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("CSO/ZSO wipe requires a readable, writable, seekable stream.", nameof(image)); + + var originalLength = image.Length; + var layout = CsoImage.ReadLayout(image); + using var staged = CsoInPlaceModifier.CreateScratchStream(); + Repack(image, staged, layout, checked((int)layout.BlockSize)); + if (staged.Length >= originalLength) + return 0; + + var reclaimed = originalLength - staged.Length; + staged.Position = 0; + image.Position = 0; + staged.CopyTo(image); + WriteZeros(image, reclaimed); + image.Flush(); + return reclaimed; + } + + /// Describes the real container byte layout; only bytes after the final index are free. + public IEnumerable EnumerateLayout(Stream archive) { + var layout = CsoImage.ReadLayout(archive); + if (layout.DataStart > 0) + yield return new DefragBlockInfo(0, layout.DataStart, DefragBlockKind.MetadataReserved, "CSO/ZSO header + index"); + + for (var i = 0; i < layout.BlockCount; ++i) { + var (offset, length, encoding) = CsoImage.GetBlockSpan(layout, i); + if (length <= 0) + continue; + yield return new DefragBlockInfo(offset, length, DefragBlockKind.Used, + $"blocks/block_{i:D5}.bin", encoding switch { + CsoBlockEncoding.Stored => DefragBlockClass.Frozen, + CsoBlockEncoding.Lz4 => DefragBlockClass.Cold, + _ => DefragBlockClass.Normal, + }); } + + if (layout.DataEnd < layout.FullSize) + yield return new DefragBlockInfo(layout.DataEnd, layout.FullSize - layout.DataEnd, DefragBlockKind.Free, "unindexed tail"); } - /// - /// "Removes" blocks by writing block_size zero bytes through - /// , which compresses the zero - /// slab to its minimum DEFLATE encoding and zero-pads the on-disk slack. - /// - public void Remove(Stream archive, string[] entryNames) { + public LayoutAnalysis AnalyzeLayout(Stream image) { + var layout = CsoImage.ReadLayout(image); + var trailing = Math.Max(0, layout.FullSize - layout.DataEnd); + var notes = new List { + $"{VariantName(layout.Variant)} with {layout.BlockCount:N0} independently compressed blocks.", + "Canonical rebuilds emit align=0 and remove index-alignment padding/orphaned data without changing logical ISO bytes.", + }; + if (layout.Align != 0) + notes.Add($"Current index_shift={layout.Align}; a rebuild can remove that alignment padding."); + return new LayoutAnalysis { + ImageSize = layout.FullSize, + CurrentUnitSize = checked((int)layout.BlockSize), + CurrentSlackBytes = trailing, + OptimalUnitSize = checked((int)layout.BlockSize), + OptimalSlackBytes = 0, + RequiresRebuild = ["BlockSize", "IndexShift"], + Notes = notes, + }; + } + + /// Reblocks and recompresses while preserving the source CSO/ZSO variant and logical ISO bytes. + public void RebuildStreaming(Stream source, Stream target, LayoutRebuildOptions options) { + ArgumentNullException.ThrowIfNull(source); + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(options); + var layout = CsoImage.ReadLayout(source); + var targetBlockSize = options.UnitSize > 0 ? options.UnitSize : checked((int)layout.BlockSize); + if (targetBlockSize <= 0) + throw new ArgumentOutOfRangeException(nameof(options), "CSO/ZSO target BlockSize must be positive."); + Repack(source, target, layout, targetBlockSize); + } + + /// Leaves a valid empty container of the same CSO/ZSO variant and block geometry. + public void Purge(Stream archive) { ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(entryNames); - var header = CsoInPlaceModifier.ReadHeader(archive); - var zero = new byte[header.BlockSize]; - foreach (var name in entryNames) { - var clean = name.Replace('\\', '/'); - var idx = ParseBlockIndex(clean); - if (idx < 0) - throw new NotSupportedException( - $"CSO: '{name}' cannot be removed. A compressed ISO is edited a block at a time, so an " - + "entry has to be named 'block_NNNNN.bin' for the block it clears. Removing a file from " - + "the ISO 9660 filesystem inside it is not something this supports."); - CsoInPlaceModifier.WriteBlock(archive, idx, zero); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("CSO/ZSO purge requires a readable, writable, seekable stream.", nameof(archive)); + var layout = CsoImage.ReadLayout(archive); + using var emptyInput = new MemoryStream([], writable: false); + using var staged = CsoInPlaceModifier.CreateScratchStream(); + CsoWriter.Write(staged, emptyInput, 0, checked((int)layout.BlockSize), layout.Variant); + Commit(staged, archive, truncate: true); + } + + private static void Repack(Stream source, Stream target, CsoLayout layout, int targetBlockSize) { + source.Position = 0; + using var logical = CsoImage.OpenLogicalStream(source, layout); + target.Position = 0; + target.SetLength(0); + CsoWriter.Write(target, logical, layout.UncompressedSize, targetBlockSize, layout.Variant); + } + + private static void Commit(Stream staged, Stream target, bool truncate) { + staged.Position = 0; + target.Position = 0; + if (truncate) + target.SetLength(0); + staged.CopyTo(target); + target.Flush(); + } + + private static void WriteZeros(Stream stream, long count) { + Span zeros = stackalloc byte[4096]; + while (count > 0) { + var length = (int)Math.Min(count, zeros.Length); + stream.Write(zeros[..length]); + count -= length; } } - /// - /// Parses the trailing block index out of names that look like - /// blocks/block_NNNNN.bin or block_NNNNN.bin. Returns -1 if - /// the name doesn't match the synthetic-entry shape. - /// + private static CsoVariant VariantFromCreateOptions(FormatCreateOptions options) { + var requested = options.GetString("Variant") ?? options.MethodName; + if (string.IsNullOrWhiteSpace(requested)) + return CsoVariant.CsoV1; + return requested.Trim().ToLowerInvariant() switch { + "cso" or "cso1" or "deflate" or "stored" => CsoVariant.CsoV1, + "zso" or "lz4" => CsoVariant.Zso, + "cso2" or "v2" => CsoVariant.CsoV2, + _ => throw new NotSupportedException($"Unsupported CSO/ZSO creation variant '{requested}'."), + }; + } + + private static bool Wants(string[]? files, string name) + => files == null || files.Length == 0 || MatchesFilter(name, files); + + private static string BuildMetadataIni(CsoLayout layout) { + var sb = new StringBuilder(); + sb.Append("[Cso]\n"); + sb.Append(CultureInfo.InvariantCulture, $"magic={layout.Magic}\n"); + sb.Append(CultureInfo.InvariantCulture, $"variant={VariantName(layout.Variant)}\n"); + sb.Append(CultureInfo.InvariantCulture, $"header_size={layout.HeaderSize}\n"); + sb.Append(CultureInfo.InvariantCulture, $"uncompressed_size={layout.UncompressedSize}\n"); + sb.Append(CultureInfo.InvariantCulture, $"block_size={layout.BlockSize}\n"); + sb.Append(CultureInfo.InvariantCulture, $"version={layout.Version}\n"); + sb.Append(CultureInfo.InvariantCulture, $"align={layout.Align}\n"); + sb.Append(CultureInfo.InvariantCulture, $"block_count={layout.BlockCount}\n"); + sb.Append(CultureInfo.InvariantCulture, $"total_size={layout.FullSize}\n"); + return sb.ToString(); + } + + private static string VariantName(CsoVariant variant) => variant switch { + CsoVariant.CsoV1 => "cso1", + CsoVariant.CsoV2 => "cso2", + CsoVariant.Zso => "zso", + _ => "unknown", + }; + + private static string MethodName(CsoBlockEncoding encoding) => encoding switch { + CsoBlockEncoding.Stored => "Stored", + CsoBlockEncoding.Deflate => "Deflate", + CsoBlockEncoding.Lz4 => "LZ4", + _ => "Unknown", + }; + + private static NotSupportedException BlockNamespaceOnly(string name, string verb) + => new( + $"CSO/ZSO: '{name}' cannot be {verb}. A compressed ISO container is edited at logical " + + "block_NNNNN.bin granularity; editing the ISO 9660 filesystem inside it is a separate operation."); + private static int ParseBlockIndex(string name) { var fileName = Path.GetFileNameWithoutExtension(name); const string prefix = "block_"; - if (!fileName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return -1; - var digits = fileName[prefix.Length..]; - return int.TryParse(digits, NumberStyles.Integer, CultureInfo.InvariantCulture, out var idx) && idx >= 0 - ? idx + if (!fileName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + return -1; + return int.TryParse(fileName[prefix.Length..], NumberStyles.Integer, CultureInfo.InvariantCulture, out var index) && index >= 0 + ? index : -1; } } From c0603f50486bca7a5495aab531343d5cb64d68e1 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:03:57 +0200 Subject: [PATCH 049/165] test(onefs): pin opaque streaming and blocked maintenance --- .../OneFs/OneFsDetectionTests.cs | 164 ++++++++++++------ 1 file changed, 109 insertions(+), 55 deletions(-) diff --git a/Compression.Tests/OneFs/OneFsDetectionTests.cs b/Compression.Tests/OneFs/OneFsDetectionTests.cs index 28588d573..e2b5b29b0 100644 --- a/Compression.Tests/OneFs/OneFsDetectionTests.cs +++ b/Compression.Tests/OneFs/OneFsDetectionTests.cs @@ -1,82 +1,136 @@ using System.Text; using Compression.Registry; +using Compression.Registry.Streaming; using FileSystem.OneFs; namespace Compression.Tests.OneFs; /// -/// Stage 0 acceptance gate for : -/// pins the detection magic, surface entry shape, and honest -/// "detection-only" Description. +/// Acceptance tests for the conservative OneFS single-image inspection surface. +/// The tests intentionally do not invent an on-disk filesystem image: Dell's +/// public material documents cluster architecture, not a standalone raw-drive +/// serialization that this suite could author independently. /// [TestFixture] public class OneFsDetectionTests { - private static byte[] BuildMinimal(int payloadLen = 128) { - var image = new byte[16 + payloadLen]; - Encoding.ASCII.GetBytes("OneFS").CopyTo(image.AsSpan(0, 5)); - for (var i = 0; i < payloadLen; i++) image[16 + i] = (byte)(i & 0xFF); + private static byte[] BuildOpaqueImage(int length = 512) { + var image = new byte[length]; + for (var i = 0; i < image.Length; ++i) + image[i] = unchecked((byte)(i * 37 + 11)); return image; } + [Test, Category("Regression")] + public void Descriptor_DoesNotAdvertiseUnverifiedMagic() { + var descriptor = new OneFsFormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor.Id, Is.EqualTo("OneFs")); + Assert.That(descriptor.Extensions, Does.Contain(".onefs")); + Assert.That(descriptor.MagicSignatures, Is.Empty, + "No Dell source was found defining 'OneFS' or 'ONEF' as an offset-zero raw-media signature."); + Assert.That(descriptor.Description, Does.Contain("signature claim was therefore removed").IgnoreCase); + }); + } + [Test, Category("HappyPath")] - public void Detector_IdentifiesByMagic() { - var d = new OneFsFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("OneFs")); - Assert.That(d.Extensions, Does.Contain(".onefs")); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(2)); - Assert.That(d.MagicSignatures[0].Bytes, Is.EqualTo("OneFS"u8.ToArray())); - Assert.That(d.MagicSignatures[1].Bytes, Is.EqualTo("ONEF"u8.ToArray())); - Assert.That(d, Is.Not.InstanceOf()); + public void List_ReturnsInspectionEntriesWithoutConsumingImage() { + var descriptor = new OneFsFormatDescriptor(); + using var image = new MemoryStream(BuildOpaqueImage(1024)); + image.Position = 137; + + var entries = descriptor.List(image, password: null); + + Assert.Multiple(() => { + Assert.That(entries.Select(entry => entry.Name), Is.EquivalentTo(new[] { + OneFsReader.MetadataEntryName, + OneFsReader.RawImageEntryName, + })); + Assert.That(entries.Single(entry => entry.Name == OneFsReader.RawImageEntryName).OriginalSize, + Is.EqualTo(image.Length)); + Assert.That(image.Position, Is.EqualTo(137), + "Listing an opaque multi-terabyte image must not copy or scan its payload."); + }); } [Test, Category("HappyPath")] - public void List_ReturnsTwoEntries() { - var d = new OneFsFormatDescriptor(); - using var ms = new MemoryStream(BuildMinimal(payloadLen: 256)); - var entries = d.List(ms, password: null); - var names = entries.Select(e => e.Name).ToList(); - Assert.That(names, Is.EquivalentTo(new[] { "metadata.ini", "onefs-volume.bin" })); + public void OpenEntry_StreamsRawImageByteExact() { + var expected = BuildOpaqueImage(4097); + using var image = new MemoryStream(expected, writable: false); + IArchiveFormatOperations operations = new OneFsFormatDescriptor(); + + using var entry = operations.OpenEntry(image, OneFsReader.RawImageEntryName, password: null); + using var copy = new MemoryStream(); + entry.CopyTo(copy); + + Assert.Multiple(() => { + Assert.That(entry, Is.InstanceOf()); + Assert.That(entry.Length, Is.EqualTo(expected.LongLength)); + Assert.That(copy.ToArray(), Is.EqualTo(expected)); + Assert.That(image.CanRead, Is.True, "The descriptor must not take ownership of the caller's archive stream."); + }); } - [Test, Category("Stub")] - public void Description_FlagsDetectionOnly() { - var d = new OneFsFormatDescriptor(); - Assert.That(d.Description.ToLowerInvariant(), Does.Contain("detection-only"), - $"OneFS Description must flag Stage 0 honestly. Got: '{d.Description}'."); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + [Test, Category("HappyPath")] + public void Metadata_ReportsOnlyDocumentedArchitecture() { + using var image = new MemoryStream(BuildOpaqueImage(777)); + using var reader = new OneFsReader(image); + var metadata = reader.Entries.Single(entry => entry.Name == OneFsReader.MetadataEntryName); + var text = Encoding.UTF8.GetString(reader.Extract(metadata)); + + Assert.Multiple(() => { + Assert.That(text, Does.Contain("parse_status=opaque-single-image")); + Assert.That(text, Does.Contain("stage=0")); + Assert.That(text, Does.Contain("authoritative_raw_magic=not_published")); + Assert.That(text, Does.Contain($"physical_block_size={OneFsReader.PhysicalBlockSize}")); + Assert.That(text, Does.Contain("LIN B+ tree")); + Assert.That(text, Does.Contain("protection groups")); + Assert.That(text, Does.Contain("rw_promotion=blocked")); + Assert.That(text, Does.Contain("maintenance=blocked")); + Assert.That(text, Does.Contain("UFS-based")); + }); } - [Test, Category("Stub")] - public void Description_NamesRoPromotionBlockers() { - // Honest Stage-0 doctrine: Description must surface WHY we cannot promote to R/O. - // Distributed/FEC-striped + no public spec are the two structural blockers; UFS - // ancestry must NOT be advertised as a fallback path. - var d = new OneFsFormatDescriptor(); - var desc = d.Description.ToLowerInvariant(); - Assert.That(desc, Does.Contain("distributed").Or.Contain("clustered"), - "OneFS Description must call out the distributed/clustered architecture."); - Assert.That(desc, Does.Contain("spec").Or.Contain("proprietary"), - "OneFS Description must call out the lack of a public on-disk specification."); - Assert.That(desc, Does.Contain("ufs"), - "OneFS Description must explicitly state UFS-incompatibility so nobody routes images through UfsReader."); + [Test, Category("Regression")] + public void LegacyMagicLookingBytes_AreTreatedAsOpaquePayload() { + var imageBytes = BuildOpaqueImage(256); + "OneFSONEF"u8.CopyTo(imageBytes); + using var image = new MemoryStream(imageBytes, writable: false); + using var reader = new OneFsReader(image); + + Assert.Multiple(() => { + Assert.That(reader.Tag, Is.Empty); + Assert.That(reader.TrailingWord, Is.Zero); + Assert.That(reader.ValidHeader, Is.False); + }); + + using var raw = reader.OpenEntry(reader.Entries.Single(entry => entry.Name == OneFsReader.RawImageEntryName)); + var prefix = new byte[9]; + raw.ReadExactly(prefix); + Assert.That(prefix, Is.EqualTo("OneFSONEF"u8.ToArray())); } [Test, Category("Stub")] - public void Metadata_NamesRoPromotionBlockers() { - // Stage-0 metadata.ini must enumerate the R/O-promotion blockers so end-users - // see them without reading source. Pin the structural reasons (distributed, - // FEC, no spec, UFS-incompatible) — not the surface phrasing. - using var ms = new MemoryStream(BuildMinimal(payloadLen: 64)); - var reader = new OneFsReader(ms); - var meta = reader.Entries.First(e => e.Name == "metadata.ini"); - var ini = Encoding.UTF8.GetString(meta.Data).ToLowerInvariant(); - Assert.That(ini, Does.Contain("stage=0"), "metadata.ini must pin stage=0."); - Assert.That(ini, Does.Contain("ro_promotion=blocked"), "metadata.ini must pin ro_promotion=blocked."); - Assert.That(ini, Does.Contain("fec"), "metadata.ini must call out FEC striping as a blocker."); - Assert.That(ini, Does.Contain("lin tree"), "metadata.ini must call out the LIN tree being cluster-wide."); - Assert.That(ini, Does.Contain("spec"), "metadata.ini must call out the missing public spec."); - Assert.That(ini, Does.Contain("ufs"), "metadata.ini must call out UFS-incompatibility at the on-disk layer."); + public void WriteAndMaintenanceCapabilities_RemainBlocked() { + var descriptor = new OneFsFormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + }); + } + + [Test, Category("Malformed")] + public void Reader_RejectsEmptyImage() { + using var image = new MemoryStream(); + Assert.That(() => new OneFsReader(image), Throws.TypeOf()); } } From 2215a169417762c028b84e92ab5c9cfc950d259d Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:02 +0200 Subject: [PATCH 050/165] Fix span-safe NRG chunk assertions --- Compression.Tests/Nrg/NrgTests.cs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/Compression.Tests/Nrg/NrgTests.cs b/Compression.Tests/Nrg/NrgTests.cs index 673cf3863..8f079d047 100644 --- a/Compression.Tests/Nrg/NrgTests.cs +++ b/Compression.Tests/Nrg/NrgTests.cs @@ -79,26 +79,30 @@ public void Create_WritesRealV2ChunkChain() { Assert.That(trailerOffset, Is.LessThan(image.Length - 12)); var position = trailerOffset; - AssertChunk(image, ref position, "ETN2", 32, payload => { - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(payload), Is.Zero); - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(payload[8..]), Is.EqualTo((ulong)trailerOffset)); - Assert.That(payload[19], Is.Zero, "cooked Mode 1"); + var etn2 = ReadChunk(image, ref position, "ETN2", 32); + Assert.Multiple(() => { + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(etn2), Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(etn2[8..]), Is.EqualTo((ulong)trailerOffset)); + Assert.That(etn2[19], Is.Zero, "cooked Mode 1"); }); - AssertChunk(image, ref position, "SINF", 4, - payload => Assert.That(BinaryPrimitives.ReadUInt32BigEndian(payload), Is.EqualTo(1))); - AssertChunk(image, ref position, "MTYP", 4, - payload => Assert.That(BinaryPrimitives.ReadUInt32BigEndian(payload), Is.EqualTo(0x00000400))); - AssertChunk(image, ref position, "END!", 0, null); + + var sinf = ReadChunk(image, ref position, "SINF", 4); + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(sinf), Is.EqualTo(1)); + + var mtyp = ReadChunk(image, ref position, "MTYP", 4); + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(mtyp), Is.EqualTo(0x00000400)); + + _ = ReadChunk(image, ref position, "END!", 0); Assert.That(position, Is.EqualTo(image.Length - 12)); } - private static void AssertChunk(byte[] image, ref int position, string id, int expectedLength, Action>? inspect) { + private static ReadOnlySpan ReadChunk(byte[] image, ref int position, string id, int expectedLength) { Assert.That(System.Text.Encoding.ASCII.GetString(image, position, 4), Is.EqualTo(id)); var length = checked((int)BinaryPrimitives.ReadUInt32BigEndian(image.AsSpan(position + 4, 4))); Assert.That(length, Is.EqualTo(expectedLength)); var payload = image.AsSpan(position + 8, length); - inspect?.Invoke(payload); position += 8 + length; + return payload; } [Test, Category("HappyPath"), Category("RoundTrip")] From aabe2fa9c3129d6e6b65f0961a2f7e1f24f02083 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:03 +0200 Subject: [PATCH 051/165] Add native T64 maintenance operations --- .../FileFormat.T64/T64FormatDescriptor.cs | 278 ++++++++++++++---- 1 file changed, 222 insertions(+), 56 deletions(-) diff --git a/FileFormats/FileFormat.T64/T64FormatDescriptor.cs b/FileFormats/FileFormat.T64/T64FormatDescriptor.cs index cc63f3348..ae9aace2a 100644 --- a/FileFormats/FileFormat.T64/T64FormatDescriptor.cs +++ b/FileFormats/FileFormat.T64/T64FormatDescriptor.cs @@ -1,4 +1,6 @@ #pragma warning disable CS1591 +using System.Buffers; +using System.Buffers.Binary; using Compression.Registry; using static Compression.Registry.FormatHelpers; @@ -10,10 +12,16 @@ namespace FileFormat.T64; /// References: /// /// Peter Schepers, "C64 File Formats: T64" — the classic reference document -/// https://vice-emu.sourceforge.io/ — VICE emulator — reference implementation reading/writing T64 +/// https://vice-emu.sourceforge.io/ — VICE emulator and T64 documentation /// /// -public sealed class T64FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable, IArchiveDefragmentable, IArchiveLayoutMap, IFilesystemBlockMover { +public sealed class T64FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, + IArchiveModifiable, IArchiveDefragmentable, IArchiveShrinkable, IArchivePurgeable, + IArchiveLayoutMap, IFilesystemBlockMover { + + private const int HeaderSize = 64; + private const int EntrySize = 32; + /// /// Gets the id. /// @@ -70,8 +78,8 @@ public sealed class T64FormatDescriptor : IFormatDescriptor, IArchiveFormatOpera /// Lists the entries in the supplied container. /// public List List(Stream stream, string? password) { - var r = new T64Reader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo( + using var reader = new T64Reader(stream); + return reader.Entries.Select((e, i) => new ArchiveEntryInfo( i, e.Name, e.Size, e.Size, "Stored", false, false, null )).ToList(); } @@ -80,10 +88,10 @@ public List List(Stream stream, string? password) { /// Decodes the supplied input. /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new T64Reader(stream); - foreach (var e in r.Entries) { - if (files != null && !MatchesFilter(e.Name, files)) continue; - WriteFile(outputDir, e.Name, r.Extract(e)); + using var reader = new T64Reader(stream); + foreach (var entry in reader.Entries) { + if (files != null && !MatchesFilter(entry.Name, files)) continue; + WriteFile(outputDir, entry.Name, reader.Extract(entry)); } } @@ -91,21 +99,16 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? /// Performs the create operation. /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { - var w = new T64Writer(); + var writer = new T64Writer(); foreach (var (name, data) in FlatFiles(inputs)) - w.AddFile(name.Length > 16 ? name[..16] : name, data); - output.Write(w.Build()); + writer.AddFile(name.Length > 16 ? name[..16] : name, data); + output.Write(writer.Build()); } // ── IArchiveModifiable (in-place) ───────────────────────────────────── /// - /// Adds (or replaces by name) files inside an existing T64 tape image via - /// . If a directory slot is free the entry - /// drops in directly and the new payload is appended at EOF. If the - /// directory is full the directory grows by one 32-byte slot — the payload - /// region shifts forward by 32 bytes and every existing slot's absolute - /// dataOffset field is patched. No full image rebuild. + /// Adds or replaces files inside an existing T64 image without rebuilding it. /// public void Add(Stream archive, IReadOnlyList inputs) { ArgumentNullException.ThrowIfNull(archive); @@ -115,11 +118,7 @@ public void Add(Stream archive, IReadOnlyList inputs) { } /// - /// Removes named entries from an existing T64 tape image via - /// . Later directory slots shift up by 32 - /// bytes, the removed payload bytes are wiped, the remaining payload region - /// shifts to close the gap (each affected slot's absolute dataOffset is - /// patched), and the stream is truncated. + /// Removes named entries, wipes their payload bytes and compacts the image. /// public void Remove(Stream archive, string[] entryNames) { ArgumentNullException.ThrowIfNull(archive); @@ -141,62 +140,229 @@ public void UpdateAllocationAfterMove(Stream image, string fileName, long oldOff // ── IArchiveDefragmentable ─────────────────────────────────────────── /// - /// Performs the defragment operation. + /// Packs live payloads immediately after the directory with native byte moves. /// public void Defragment(Stream archive) => Defragment(archive, new DefragOptions { Mode = DefragMode.ConsolidateAtStart }); /// - /// Defragments a T64 image. Falls back to rebuild since T64 data offsets - /// are stored in directory entries and recompaction is simplest via rebuild. + /// Defragments a T64 image. The canonical consolidate-at-start path moves + /// payloads in place and patches only their absolute data-offset fields. + /// Other layout modes retain the established rebuild behaviour, but preserve + /// T64 version, tape name, load address and Commodore file type. + /// + public void Defragment(Stream archive, DefragOptions options) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(options); + EnsureMutable(archive); + options.CancellationToken.ThrowIfCancellationRequested(); + + if (options.Mode != DefragMode.ConsolidateAtStart) { + DefragmentByRebuild(archive, options); + return; + } + + archive.Position = 0; + using var reader = new T64Reader(archive); + var entries = reader.Entries.OrderBy(static e => e.DataOffset).ToArray(); + var imageSize = archive.Length; + var cursor = (long)HeaderSize + reader.DirectoryEntryCount * EntrySize; + var mover = new T64BlockMover(); + + options.OnProgress?.Invoke(new DefragProgressEvent( + Phase: "scanning", Fraction: 0, CurrentReadOffset: 0, CurrentWriteOffset: -1, + ImageSize: imageSize, BlockMap: EnumerateLayout(archive).ToList(), Status: "Analysing T64 layout")); + + for (var i = 0; i < entries.Length; i++) { + options.CancellationToken.ThrowIfCancellationRequested(); + var entry = entries[i]; + if (entry.DataOffset != cursor) { + if (entry.Size > 0) + mover.MoveExtent(archive, entry.DataOffset, cursor, entry.Size, zeroSource: true); + mover.UpdateAllocationAfterMove(archive, entry.Name, entry.DataOffset, cursor, entry.Size); + } + cursor += entry.Size; + + options.OnProgress?.Invoke(new DefragProgressEvent( + Phase: "moving", Fraction: entries.Length == 0 ? 1 : (double)(i + 1) / entries.Length, + CurrentReadOffset: entry.DataOffset, CurrentWriteOffset: cursor, + ImageSize: imageSize, BlockMap: null, Status: $"Packed {entry.Name}")); + } + + archive.SetLength(cursor); + archive.Position = 0; + options.OnProgress?.Invoke(new DefragProgressEvent( + Phase: "complete", Fraction: 1, CurrentReadOffset: -1, CurrentWriteOffset: -1, + ImageSize: archive.Length, BlockMap: EnumerateLayout(archive).ToList(), Status: "Defragmentation complete")); + } + + // ── IArchiveShrinkable ─────────────────────────────────────────────── + + /// + /// Rebuilds normal T64 records into the smallest byte-tight image while + /// preserving version, tape name, start addresses and Commodore file types. + /// Images containing non-normal record kinds are copied through unchanged. + /// + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("T64: shrink input must be readable and seekable.", nameof(input)); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("T64: shrink output must be writable and seekable.", nameof(output)); + + input.Position = 0; + using var reader = new T64Reader(input); + if (reader.Entries.Any(static e => e.EntryType != 1)) { + CopyOriginal(input, output); + return; + } + + var rebuilt = BuildCompactImage(reader); + if (rebuilt.LongLength >= input.Length) { + CopyOriginal(input, output); + return; + } + + output.Position = 0; + output.SetLength(0); + output.Write(rebuilt); + output.Position = 0; + } + + // ── IArchivePurgeable ──────────────────────────────────────────────── + + /// + /// Wipes all directory/payload bytes and leaves the canonical 64-byte empty T64 header. /// - public void Defragment(Stream archive, DefragOptions options) - => DefragRebuilder.Rebuild(archive, options, ReadEntries, BuildImage); + public void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + EnsureMutable(archive); + + archive.Position = 0; + using (var reader = new T64Reader(archive)) { } + + if (archive.Length > HeaderSize) + ZeroRange(archive, HeaderSize, archive.Length - HeaderSize); + + Span counts = stackalloc byte[4]; + archive.Position = 34; + archive.Write(counts); + archive.SetLength(HeaderSize); + archive.Position = 0; + } - // ── IArchiveLayoutMap ──────────────────────────────────────────────── + // ── IArchiveLayoutMap / IWipeEmpty ────────────────────────────────── /// - /// Enumerates the byte layout of a T64 tape image: 64-byte header as - /// MetadataReserved, N×32-byte directory entries as MetadataReserved, - /// and each file's data region as Used. + /// Enumerates every byte of a T64 image: the fixed header, live and free + /// directory slots, live payloads, and proven dead gaps/tail bytes. /// public IEnumerable EnumerateLayout(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); archive.Position = 0; - var r = new T64Reader(archive); + using var reader = new T64Reader(archive); + var liveBySlot = reader.Entries.ToDictionary(static e => e.DirectoryIndex); - // Header: 64 bytes - const int headerSize = 64; - const int entrySize = 32; - var dirSize = r.Entries.Count * entrySize; + yield return new DefragBlockInfo(0, HeaderSize, DefragBlockKind.MetadataReserved, "T64 Header"); + + for (var i = 0; i < reader.DirectoryEntryCount; i++) { + var offset = HeaderSize + (long)i * EntrySize; + if (liveBySlot.TryGetValue(i, out var entry)) + yield return new DefragBlockInfo(offset, EntrySize, DefragBlockKind.MetadataReserved, $"Directory: {entry.Name}"); + else + yield return new DefragBlockInfo(offset, EntrySize, DefragBlockKind.Free); + } - yield return new DefragBlockInfo(0, headerSize, DefragBlockKind.MetadataReserved, "T64 Header"); - if (dirSize > 0) - yield return new DefragBlockInfo(headerSize, dirSize, DefragBlockKind.MetadataReserved, "Directory"); + var cursor = (long)HeaderSize + reader.DirectoryEntryCount * EntrySize; + foreach (var entry in reader.Entries.OrderBy(static e => e.DataOffset)) { + if (entry.DataOffset > cursor) + yield return new DefragBlockInfo(cursor, entry.DataOffset - cursor, DefragBlockKind.Free); - foreach (var e in r.Entries) { - if (e.Size > 0) - yield return new DefragBlockInfo(e.DataOffset, e.Size, DefragBlockKind.Used, e.Name); + if (entry.Size > 0) + yield return new DefragBlockInfo(entry.DataOffset, entry.Size, DefragBlockKind.Used, entry.Name); + + cursor = Math.Max(cursor, entry.DataOffset + entry.Size); } - var imageSize = archive.Length; - var dataEnd = r.Entries.Count > 0 - ? r.Entries.Max(e => e.DataOffset + e.Size) - : headerSize + dirSize; - if (dataEnd < imageSize) - yield return new DefragBlockInfo(dataEnd, imageSize - dataEnd, DefragBlockKind.Free); + if (cursor < archive.Length) + yield return new DefragBlockInfo(cursor, archive.Length - cursor, DefragBlockKind.Free); } - // ── Shared delegates ───────────────────────────────────────────────── + private static void DefragmentByRebuild(Stream archive, DefragOptions options) { + archive.Position = 0; + using var reader = new T64Reader(archive); + if (reader.Entries.Any(static e => e.EntryType != 1)) + throw new NotSupportedException("T64 rebuild defrag only supports normal directory records; consolidate-at-start can move other record kinds in place."); + + var tapeName = reader.TapeName; + var version = reader.Version; + var metadata = reader.Entries.Select(e => new SnapshotEntry( + e.Name, e.StartAddress, e.FileType, reader.Extract(e))).ToList(); + + DefragRebuilder.Rebuild(archive, options, ReadEntries, files => { + var remaining = metadata.ToList(); + var writer = new T64Writer(); + foreach (var (name, data) in files) { + var index = remaining.FindIndex(e => + e.Name.Equals(name, StringComparison.Ordinal) && e.Data.AsSpan().SequenceEqual(data)); + if (index < 0) { + writer.AddFile(name, data); + continue; + } + + var source = remaining[index]; + remaining.RemoveAt(index); + writer.AddFile(name, source.StartAddress, source.FileType, data); + } + return writer.Build(tapeName, version); + }); + } private static IEnumerable<(string Name, byte[] Data)> ReadEntries(Stream stream) { - var r = new T64Reader(stream); - return r.Entries.Select(e => (e.Name, r.Extract(e))); + using var reader = new T64Reader(stream); + return reader.Entries.Select(e => (e.Name, reader.Extract(e))).ToArray(); + } + + private static byte[] BuildCompactImage(T64Reader reader) { + var writer = new T64Writer(); + foreach (var entry in reader.Entries) + writer.AddFile(entry.Name, entry.StartAddress, entry.FileType, reader.Extract(entry)); + return writer.Build(reader.TapeName, reader.Version); + } + + private static void CopyOriginal(Stream input, Stream output) { + if (ReferenceEquals(input, output)) { + input.Position = 0; + return; + } + + input.Position = 0; + output.Position = 0; + output.SetLength(0); + input.CopyTo(output); + output.Position = 0; } - private static byte[] BuildImage(IReadOnlyList<(string Name, byte[] Data)> files) { - var w = new T64Writer(); - foreach (var (n, d) in files) - w.AddFile(n.Length > 16 ? n[..16] : n, d); - return w.Build(); + private static void EnsureMutable(Stream archive) { + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("T64: stream must be readable, writable and seekable.", nameof(archive)); } + + private static void ZeroRange(Stream stream, long offset, long length) { + var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); + try { + Array.Clear(buffer, 0, buffer.Length); + stream.Position = offset; + while (length > 0) { + var chunk = (int)Math.Min(length, buffer.Length); + stream.Write(buffer, 0, chunk); + length -= chunk; + } + } finally { + ArrayPool.Shared.Return(buffer); + } + } + + private sealed record SnapshotEntry(string Name, ushort StartAddress, byte FileType, byte[] Data); } From 8f4e4d1bb743c69db6c6c14d993c795307f2347a Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:03 +0200 Subject: [PATCH 052/165] Batch CSO block replacements in one repack --- .../FileFormat.Cso/CsoInPlaceModifier.cs | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs b/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs index d860ab836..d205fa3d9 100644 --- a/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs +++ b/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs @@ -17,22 +17,29 @@ public static class CsoInPlaceModifier { /// Replaces one logical block. must be exactly block_size /// bytes; for a partial final block only its logical prefix is retained, matching CSO semantics. /// - public static void WriteBlock(Stream image, int blockIndex, ReadOnlySpan newUncompressedData) { + public static void WriteBlock(Stream image, int blockIndex, ReadOnlySpan newUncompressedData) + => WriteBlocks(image, new Dictionary { [blockIndex] = newUncompressedData.ToArray() }); + + /// Replaces several logical blocks in one transactional repack. + internal static void WriteBlocks(Stream image, IReadOnlyDictionary replacements) { ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(replacements); if (!image.CanRead || !image.CanWrite || !image.CanSeek) throw new InvalidOperationException("CSO/ZSO block modification requires a readable, writable, seekable stream."); var layout = CsoImage.ReadLayout(image); - if ((uint)blockIndex >= (uint)layout.BlockCount) - throw new ArgumentOutOfRangeException(nameof(blockIndex), - $"CSO/ZSO block index {blockIndex} outside [0, {layout.BlockCount})."); - if (newUncompressedData.Length != layout.BlockSize) - throw new ArgumentException( - $"New block payload must be exactly block_size ({layout.BlockSize}) bytes; got {newUncompressedData.Length}.", - nameof(newUncompressedData)); - - var replacement = newUncompressedData.ToArray(); - var replacements = new Dictionary { [blockIndex] = replacement }; + foreach (var (blockIndex, replacement) in replacements) { + if ((uint)blockIndex >= (uint)layout.BlockCount) + throw new ArgumentOutOfRangeException(nameof(replacements), + $"CSO/ZSO block index {blockIndex} outside [0, {layout.BlockCount})."); + if (replacement.Length != checked((int)layout.BlockSize)) + throw new ArgumentException( + $"New block payload must be exactly block_size ({layout.BlockSize}) bytes; got {replacement.Length} for block {blockIndex}.", + nameof(replacements)); + } + if (replacements.Count == 0) + return; + using var logical = CsoImage.OpenLogicalStream(image, layout, replacements); using var staged = CreateScratchStream(); CsoWriter.Write(staged, logical, layout.UncompressedSize, checked((int)layout.BlockSize), layout.Variant); From 6e3c0c54ffcb0a07d86d2297733717ed95d13d88 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:14 +0200 Subject: [PATCH 053/165] Add GlusterFS backing filesystem dependencies --- FileSystems/FileSystem.GlusterFs/FileSystem.GlusterFs.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/FileSystems/FileSystem.GlusterFs/FileSystem.GlusterFs.csproj b/FileSystems/FileSystem.GlusterFs/FileSystem.GlusterFs.csproj index 78cd2b68f..24a7d9e51 100644 --- a/FileSystems/FileSystem.GlusterFs/FileSystem.GlusterFs.csproj +++ b/FileSystems/FileSystem.GlusterFs/FileSystem.GlusterFs.csproj @@ -4,5 +4,7 @@ + + From 20f6b82874ef59754e89f6cd10b4d1dd144fd478 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:15 +0200 Subject: [PATCH 054/165] Add conservative TUX3 layout, wipe, and shrink support --- .../FileSystem.Tux3/Tux3FormatDescriptor.cs | 106 +++++++++++++++++- 1 file changed, 103 insertions(+), 3 deletions(-) diff --git a/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs b/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs index 63aaff81d..9aac1144f 100644 --- a/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs +++ b/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs @@ -5,15 +5,23 @@ namespace FileSystem.Tux3; /// -/// Read-only native-superblock descriptor for the linux-tux3 research filesystem. +/// Native-superblock descriptor for the linux-tux3 research filesystem. /// /// /// The descriptor recognises real linux-tux3 disk-format revisions and parses the packed, /// big-endian struct disksuper at byte 4096. Native tree traversal and mutation are not /// implemented, so Create/Modify/Defragment capabilities are intentionally withheld rather than /// routing files through a private side-table that no TUX3 implementation understands. +/// +/// The on-disk volblocks field does, however, define the native volume boundary. +/// Bytes after that boundary are outside the TUX3 volume and can therefore be reported as free, +/// wiped, and removed by shrink without interpreting or modifying any native tree. Everything +/// inside the declared volume remains fail-closed as metadata-reserved until allocation-tree +/// traversal exists. /// -public sealed class Tux3FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, ISyntheticEntryNames { +public sealed class Tux3FormatDescriptor : + IFormatDescriptor, IArchiveFormatOperations, ISyntheticEntryNames, IFilesystemExtentMap, IArchiveShrinkable { + private static readonly HashSet SyntheticNames = new(StringComparer.OrdinalIgnoreCase) { "FULL.tux3", "metadata.ini", "superblock.bin" }; @@ -59,7 +67,8 @@ public sealed class Tux3FormatDescriptor : IFormatDescriptor, IArchiveFormatOper /// public string Description => - "TUX3 version-tree research filesystem — native big-endian superblock detection/metadata; tree traversal and writing not yet implemented."; + "TUX3 version-tree research filesystem — native big-endian superblock detection/metadata, " + + "fail-closed layout mapping, external-padding wipe/shrink; native tree traversal and writing not yet implemented."; /// public List List(Stream stream, string? password) { @@ -81,4 +90,95 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? reader.ExtractTo(entry, output); } } + + /// + /// Enumerates the provable byte layout without guessing at undecoded TUX3 allocation state. + /// The declared native volume is reserved wholesale; only trailing bytes outside it are free. + /// + public IEnumerable EnumerateExtents(Stream image) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanSeek) + return []; + + try { + using var reader = new Tux3Reader(image); + if (!TryGetDeclaredVolumeLength(reader, out var volumeLength)) + return []; + + var imageLength = image.Length; + if (volumeLength >= imageLength) + return imageLength == 0 + ? [] + : [new DefragBlockInfo(0, imageLength, DefragBlockKind.MetadataReserved, "TUX3 volume (allocation map unresolved)")]; + + return [ + new DefragBlockInfo(0, volumeLength, DefragBlockKind.MetadataReserved, "TUX3 volume (allocation map unresolved)"), + new DefragBlockInfo(volumeLength, imageLength - volumeLength, DefragBlockKind.Free, "Trailing bytes outside TUX3 volume"), + ]; + } catch (InvalidDataException) { + return []; + } catch (IOException) { + return []; + } + } + + /// + /// Removes only bytes beyond the volume size declared by volblocks << blockbits. + /// Malformed, truncated, or arithmetically invalid images are copied through unchanged. + /// + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("TUX3 shrink requires a readable, seekable input stream.", nameof(input)); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("TUX3 shrink requires a writable, seekable output stream.", nameof(output)); + if (ReferenceEquals(input, output)) + throw new ArgumentException("TUX3 shrink requires distinct input and output streams.", nameof(output)); + + var copyLength = input.Length; + try { + using var reader = new Tux3Reader(input); + if (TryGetDeclaredVolumeLength(reader, out var volumeLength) && volumeLength <= input.Length) + copyLength = volumeLength; + } catch (InvalidDataException) { + // Total operation: malformed input is copied through unchanged. + } catch (IOException) { + // Total operation: an unreadable metadata surface is copied through unchanged. + } + + input.Position = 0; + output.Position = 0; + output.SetLength(0); + CopyPrefix(input, output, copyLength); + } + + private static bool TryGetDeclaredVolumeLength(Tux3Reader reader, out long length) { + length = 0; + if (!reader.ValidSuperblock || reader.VolBlocks == 0 || reader.BlockBits >= 63) + return false; + + var blockSize = 1UL << reader.BlockBits; + if (reader.VolBlocks > (ulong)long.MaxValue / blockSize) + return false; + + var declared = reader.VolBlocks * blockSize; + if (declared < Tux3Reader.SuperblockOffset + Tux3Reader.DiskSuperSize) + return false; + + length = (long)declared; + return true; + } + + private static void CopyPrefix(Stream input, Stream output, long count) { + var buffer = new byte[128 * 1024]; + while (count > 0) { + var requested = (int)Math.Min(buffer.Length, count); + var read = input.Read(buffer, 0, requested); + if (read <= 0) + throw new EndOfStreamException("TUX3 input ended before the expected image boundary."); + output.Write(buffer, 0, read); + count -= read; + } + } } From d5c82ed9f13a06376bbc62163092c894deae2371 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:24 +0200 Subject: [PATCH 055/165] Allow lazy GlusterFS brick entry extraction --- FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs b/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs index 73195b9d2..acbf0813d 100644 --- a/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs +++ b/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs @@ -2,7 +2,7 @@ namespace FileSystem.GlusterFs; /// -/// Represents a gluster fs entry. +/// Represents an entry surfaced from a single GlusterFS brick backing store. /// public sealed class GlusterFsEntry { /// @@ -22,7 +22,12 @@ public sealed class GlusterFsEntry { /// public long Offset { get; init; } /// - /// Gets or sets the data. + /// Gets or sets eagerly materialized data, used by synthetic metadata entries. /// public byte[] Data { get; init; } = []; + + /// + /// Lazily materializes an entry delegated to the backing filesystem reader. + /// + internal Func? DataFactory { get; init; } } From 36f67cbaf911b4d3b7c98d01f5316d0472c61423 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:27 +0200 Subject: [PATCH 056/165] Align NRG low-level edit tests with chunked images --- .../Nrg/NrgInPlaceModifyTests.cs | 521 ++---------------- 1 file changed, 58 insertions(+), 463 deletions(-) diff --git a/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs b/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs index 1818877dc..ddd3f6fd6 100644 --- a/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs +++ b/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs @@ -1,4 +1,5 @@ #pragma warning disable CS1591 +using System.Buffers.Binary; using Compression.Registry; using FileFormat.Nrg; @@ -6,491 +7,85 @@ namespace Compression.Tests.Nrg; [TestFixture] public class NrgInPlaceModifyTests { + private const int SectorSize = 2048; - private const int Iso9660SectorSize = 2048; - private const int RawSectorSize = 2352; - private const int Mode1DataOffset = 16; - private const int Ner5FooterSize = 12; - private const int NeroFooterSize = 8; - - // ── geometry fixtures ─────────────────────────────────────────────── - - /// - /// Builds a raw 2 352-byte/sector Mode 1 NRG v2 image: PVD at LBA 16 with - /// CD001 + 12-byte "NER5" footer at EOF. - /// - private static byte[] BuildRawMode1Nrg(int sectorCount = 32) { - if (sectorCount <= 16) sectorCount = 20; - var dataLen = sectorCount * RawSectorSize; - var buf = new byte[dataLen + Ner5FooterSize]; - - for (var i = 0; i < dataLen; i++) - buf[i] = (byte)((i * 31 + 7) & 0xFF); - - Span sync = stackalloc byte[12] { - 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, - }; - for (var s = 0; s < sectorCount; s++) { - sync.CopyTo(buf.AsSpan(s * RawSectorSize, 12)); - buf[s * RawSectorSize + 12] = 0; - buf[s * RawSectorSize + 13] = 0; - buf[s * RawSectorSize + 14] = (byte)s; - buf[s * RawSectorSize + 15] = 0x01; - } - - var pvdAt = 16 * RawSectorSize + Mode1DataOffset; - buf[pvdAt + 0] = 1; - buf[pvdAt + 1] = (byte)'C'; - buf[pvdAt + 2] = (byte)'D'; - buf[pvdAt + 3] = (byte)'0'; - buf[pvdAt + 4] = (byte)'0'; - buf[pvdAt + 5] = (byte)'1'; - buf[pvdAt + 6] = 1; - - // NER5 footer: "NER5" + uint64 BE chunk-table offset (points at end of data). - buf[dataLen + 0] = (byte)'N'; - buf[dataLen + 1] = (byte)'E'; - buf[dataLen + 2] = (byte)'R'; - buf[dataLen + 3] = (byte)'5'; - System.Buffers.Binary.BinaryPrimitives.WriteUInt64BigEndian( - buf.AsSpan(dataLen + 4, 8), (ulong)dataLen); - - return buf; - } - - /// - /// Builds a cooked 2 048-byte/sector NRG v2 image. - /// - private static byte[] BuildCookedNrg(int sectorCount = 32) { - if (sectorCount <= 16) sectorCount = 20; - var dataLen = sectorCount * Iso9660SectorSize; - var buf = new byte[dataLen + Ner5FooterSize]; - - for (var i = 0; i < dataLen; i++) - buf[i] = (byte)((i * 17 + 3) & 0xFF); - - var pvdAt = 16 * Iso9660SectorSize; - buf[pvdAt + 0] = 1; - buf[pvdAt + 1] = (byte)'C'; - buf[pvdAt + 2] = (byte)'D'; - buf[pvdAt + 3] = (byte)'0'; - buf[pvdAt + 4] = (byte)'0'; - buf[pvdAt + 5] = (byte)'1'; - buf[pvdAt + 6] = 1; - - buf[dataLen + 0] = (byte)'N'; - buf[dataLen + 1] = (byte)'E'; - buf[dataLen + 2] = (byte)'R'; - buf[dataLen + 3] = (byte)'5'; - System.Buffers.Binary.BinaryPrimitives.WriteUInt64BigEndian( - buf.AsSpan(dataLen + 4, 8), (ulong)dataLen); - return buf; - } - - /// - /// Builds a raw NRG v1 image: 8-byte "NERO" footer at EOF. - /// - private static byte[] BuildRawMode1NrgV1(int sectorCount = 32) { - if (sectorCount <= 16) sectorCount = 20; - var dataLen = sectorCount * RawSectorSize; - var buf = new byte[dataLen + NeroFooterSize]; - - for (var i = 0; i < dataLen; i++) - buf[i] = (byte)((i * 31 + 7) & 0xFF); - - Span sync = stackalloc byte[12] { - 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, - }; - for (var s = 0; s < sectorCount; s++) { - sync.CopyTo(buf.AsSpan(s * RawSectorSize, 12)); - buf[s * RawSectorSize + 14] = (byte)s; - buf[s * RawSectorSize + 15] = 0x01; - } - - var pvdAt = 16 * RawSectorSize + Mode1DataOffset; - buf[pvdAt + 0] = 1; - buf[pvdAt + 1] = (byte)'C'; - buf[pvdAt + 2] = (byte)'D'; - buf[pvdAt + 3] = (byte)'0'; - buf[pvdAt + 4] = (byte)'0'; - buf[pvdAt + 5] = (byte)'1'; - buf[pvdAt + 6] = 1; - - buf[dataLen + 0] = (byte)'N'; - buf[dataLen + 1] = (byte)'E'; - buf[dataLen + 2] = (byte)'R'; - buf[dataLen + 3] = (byte)'O'; - System.Buffers.Binary.BinaryPrimitives.WriteUInt32BigEndian( - buf.AsSpan(dataLen + 4, 4), (uint)dataLen); - return buf; - } - - // ── WriteSector — happy path ───────────────────────────────────────── - - [Test, Category("RoundTrip")] - public void WriteSector_Raw_RewritesUserDataOnly_FramingPreserved() { - var image = BuildRawMode1Nrg(); - var original = (byte[])image.Clone(); - using var ms = new MemoryStream(image, writable: true); - - var payload = new byte[Iso9660SectorSize]; - for (var i = 0; i < payload.Length; i++) payload[i] = (byte)((i + 1) & 0xFF); - - NrgInPlaceModifier.WriteSector(ms, lba: 20, payload); - - var userDataAt = 20 * RawSectorSize + Mode1DataOffset; - Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); - - Assert.That(image.AsSpan(20 * RawSectorSize, 16).ToArray(), - Is.EqualTo(original.AsSpan(20 * RawSectorSize, 16).ToArray())); - - Assert.That(image.AsSpan(userDataAt + Iso9660SectorSize, - RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray(), - Is.EqualTo(original.AsSpan(userDataAt + Iso9660SectorSize, - RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray())); - - // NER5 footer byte-identical. - Assert.That(image.AsSpan(image.Length - Ner5FooterSize, Ner5FooterSize).ToArray(), - Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize, Ner5FooterSize).ToArray())); + private static byte[] BuildNrg() { + var descriptor = new NrgFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create( + image, + [ArchiveInputInfo.InMemory("DATA.BIN", Enumerable.Range(0, 4096).Select(static i => (byte)i).ToArray())], + new FormatCreateOptions()); + return image.ToArray(); } - [Test, Category("RoundTrip")] - public void WriteSector_Raw_OtherSectors_ByteIdentical() { - var image = BuildRawMode1Nrg(); - var original = (byte[])image.Clone(); - using var ms = new MemoryStream(image, writable: true); - - var payload = new byte[Iso9660SectorSize]; - Array.Fill(payload, (byte)0xA5); - - NrgInPlaceModifier.WriteSector(ms, lba: 20, payload); - - var sectorCount = (image.Length - Ner5FooterSize) / RawSectorSize; - for (var lba = 0; lba < sectorCount; lba++) { - if (lba == 20) continue; - var sectorOff = lba * RawSectorSize; - Assert.That(image.AsSpan(sectorOff, RawSectorSize).ToArray(), - Is.EqualTo(original.AsSpan(sectorOff, RawSectorSize).ToArray()), - $"LBA {lba} unexpectedly changed."); - } - } - - [Test, Category("RoundTrip")] - public void WriteSector_Cooked_RewritesUserDataAtSectorOffset() { - var image = BuildCookedNrg(); - var original = (byte[])image.Clone(); - using var ms = new MemoryStream(image, writable: true); - - var payload = new byte[Iso9660SectorSize]; - for (var i = 0; i < payload.Length; i++) payload[i] = (byte)(i & 0xFF); - - NrgInPlaceModifier.WriteSector(ms, lba: 22, payload); - - var sectorAt = 22 * Iso9660SectorSize; - Assert.That(image.AsSpan(sectorAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); - - Assert.That(image.AsSpan(0, sectorAt).ToArray(), - Is.EqualTo(original.AsSpan(0, sectorAt).ToArray())); - Assert.That(image.AsSpan(sectorAt + Iso9660SectorSize).ToArray(), - Is.EqualTo(original.AsSpan(sectorAt + Iso9660SectorSize).ToArray())); - } - - // ── WriteSector — append path ──────────────────────────────────────── - - [Test, Category("RoundTrip")] - public void WriteSector_PastEof_AppendsWithCorrectFraming_FooterRelocated() { - var image = BuildRawMode1Nrg(sectorCount: 20); - var original = (byte[])image.Clone(); - using var ms = new MemoryStream(); - ms.Write(image); - - var payload = new byte[Iso9660SectorSize]; - Array.Fill(payload, (byte)0x5A); - - NrgInPlaceModifier.WriteSector(ms, lba: 25, payload); - - var grown = ms.ToArray(); - Assert.That(grown.Length, Is.EqualTo(26 * RawSectorSize + Ner5FooterSize)); + private static int TrailerOffset(byte[] image) + => checked((int)BinaryPrimitives.ReadUInt64BigEndian(image.AsSpan(image.Length - 8))); - Assert.That(grown.AsSpan(0, 20 * RawSectorSize).ToArray(), - Is.EqualTo(original.AsSpan(0, 20 * RawSectorSize).ToArray())); + [Test, Category("Regression")] + public void DetectGeometry_UsesDescriptorOffset_NotFooterOffset() { + var image = BuildNrg(); + using var stream = new MemoryStream(image, writable: true); - var userDataAt = 25 * RawSectorSize + Mode1DataOffset; - Assert.That(grown.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); + var geometry = NrgInPlaceModifier.DetectGeometry(stream); - for (var lba = 20; lba <= 24; lba++) { - var sectorOff = lba * RawSectorSize; - Assert.That(grown[sectorOff + 0], Is.EqualTo(0x00)); - Assert.That(grown[sectorOff + 1], Is.EqualTo(0xFF)); - Assert.That(grown[sectorOff + 15], Is.EqualTo(0x01), $"LBA {lba} mode byte"); - } - - Assert.That(grown.AsSpan(grown.Length - Ner5FooterSize, Ner5FooterSize).ToArray(), - Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize, Ner5FooterSize).ToArray())); + Assert.Multiple(() => { + Assert.That(geometry.SectorSize, Is.EqualTo(SectorSize)); + Assert.That(geometry.DataOffset, Is.Zero); + Assert.That(geometry.DataAreaLength, Is.EqualTo(TrailerOffset(image))); + Assert.That(geometry.DataAreaLength, Is.LessThan(image.Length - 12), "chunk table is not sector data"); + }); } - // ── ZeroSector ────────────────────────────────────────────────────── - [Test, Category("RoundTrip")] - public void ZeroSector_WipesUserData_PreservesFramingAndFooter() { - var image = BuildRawMode1Nrg(); - var original = (byte[])image.Clone(); - using var ms = new MemoryStream(image, writable: true); - - Assert.That(NrgInPlaceModifier.ZeroSector(ms, lba: 19), Is.True); + public void WriteSector_ExistingCookedSector_PreservesWholeDescriptor() { + var image = BuildNrg(); + var trailer = TrailerOffset(image); + var descriptorBytes = image.AsSpan(trailer).ToArray(); + using var stream = new MemoryStream(image, writable: true); + var payload = Enumerable.Range(0, SectorSize).Select(static i => (byte)(i * 29)).ToArray(); - var userDataAt = 19 * RawSectorSize + Mode1DataOffset; - Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), - Is.EqualTo(new byte[Iso9660SectorSize])); + NrgInPlaceModifier.WriteSector(stream, 20, payload); - Assert.That(image.AsSpan(19 * RawSectorSize, 16).ToArray(), - Is.EqualTo(original.AsSpan(19 * RawSectorSize, 16).ToArray())); - - Assert.That(image.AsSpan(userDataAt + Iso9660SectorSize, - RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray(), - Is.EqualTo(original.AsSpan(userDataAt + Iso9660SectorSize, - RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray())); - - Assert.That(image.AsSpan(image.Length - Ner5FooterSize, Ner5FooterSize).ToArray(), - Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize, Ner5FooterSize).ToArray())); + Assert.That(image.AsSpan(20 * SectorSize, SectorSize).ToArray(), Is.EqualTo(payload)); + Assert.That(image.AsSpan(trailer).ToArray(), Is.EqualTo(descriptorBytes)); } [Test, Category("RoundTrip")] - public void ZeroSector_PastEof_ReturnsFalse() { - var image = BuildRawMode1Nrg(); - using var ms = new MemoryStream(image, writable: true); - Assert.That(NrgInPlaceModifier.ZeroSector(ms, lba: 9999), Is.False); - } + public void ZeroSector_ExistingCookedSector_PreservesWholeDescriptor() { + var image = BuildNrg(); + var trailer = TrailerOffset(image); + var descriptorBytes = image.AsSpan(trailer).ToArray(); + using var stream = new MemoryStream(image, writable: true); - // ── NRG v1 (NERO) footer ─────────────────────────────────────────── + Assert.That(NrgInPlaceModifier.ZeroSector(stream, 20), Is.True); - [Test, Category("RoundTrip")] - public void DetectGeometry_StripsNeroV1Footer_FromDataAreaLength() { - var image = BuildRawMode1NrgV1(sectorCount: 32); - using var ms = new MemoryStream(image, writable: true); - var geom = NrgInPlaceModifier.DetectGeometry(ms); - Assert.That(geom.SectorSize, Is.EqualTo(RawSectorSize)); - Assert.That(geom.DataOffset, Is.EqualTo(Mode1DataOffset)); - Assert.That(geom.DataAreaLength, Is.EqualTo(32 * RawSectorSize)); + Assert.That(image.AsSpan(20 * SectorSize, SectorSize).ToArray(), Is.EqualTo(new byte[SectorSize])); + Assert.That(image.AsSpan(trailer).ToArray(), Is.EqualTo(descriptorBytes)); } - [Test, Category("RoundTrip")] - public void WriteSector_NeroV1_FooterPreserved() { - var image = BuildRawMode1NrgV1(); - var original = (byte[])image.Clone(); - using var ms = new MemoryStream(image, writable: true); - - var payload = new byte[Iso9660SectorSize]; - Array.Fill(payload, (byte)0x33); - - NrgInPlaceModifier.WriteSector(ms, lba: 20, payload); + [Test, Category("Regression")] + public void WriteSector_PastTrackEnd_RefusesInsteadOfCorruptingChunkOffsets() { + var image = BuildNrg(); + var trailer = TrailerOffset(image); + using var stream = new MemoryStream(); + stream.Write(image); + stream.Position = 0; - var userDataAt = 20 * RawSectorSize + Mode1DataOffset; - Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); + var firstSectorPastTrack = trailer / SectorSize; + var payload = new byte[SectorSize]; - Assert.That(image.AsSpan(image.Length - NeroFooterSize, NeroFooterSize).ToArray(), - Is.EqualTo(original.AsSpan(original.Length - NeroFooterSize, NeroFooterSize).ToArray())); + Assert.That( + () => NrgInPlaceModifier.WriteSector(stream, firstSectorPastTrack, payload), + Throws.TypeOf()); + Assert.That(stream.ToArray(), Is.EqualTo(image)); } - // ── Descriptor-level Add/Remove ───────────────────────────────────── - - [Test, Category("RoundTrip")] - public void Descriptor_Add_RewritesNamedSector() { - var image = BuildRawMode1Nrg(); - var original = (byte[])image.Clone(); - using var ms = new MemoryStream(image, writable: true); - - var payload = new byte[Iso9660SectorSize]; - for (var i = 0; i < payload.Length; i++) payload[i] = (byte)(i % 251); - - ((IArchiveModifiable)new NrgFormatDescriptor()).Add(ms, [ - ArchiveInputInfo.InMemory(NrgInPlaceModifier.FormatSectorEntryName(20), payload), - ]); - - var userDataAt = 20 * RawSectorSize + Mode1DataOffset; - Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); - - var sectorCount = (image.Length - Ner5FooterSize) / RawSectorSize; - for (var lba = 0; lba < sectorCount; lba++) { - if (lba == 20) continue; - var off = lba * RawSectorSize; - Assert.That(image.AsSpan(off, RawSectorSize).ToArray(), - Is.EqualTo(original.AsSpan(off, RawSectorSize).ToArray()), - $"LBA {lba} unexpectedly mutated."); - } - - Assert.That(image.AsSpan(image.Length - Ner5FooterSize, Ner5FooterSize).ToArray(), - Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize, Ner5FooterSize).ToArray())); - } - - [Test, Category("RoundTrip")] - public void Descriptor_Add_AppendsPastEofPreservingDataAndFooter() { - var image = BuildRawMode1Nrg(sectorCount: 20); - var original = (byte[])image.Clone(); - using var ms = new MemoryStream(); - ms.Write(image); - - var payload = new byte[Iso9660SectorSize]; - Array.Fill(payload, (byte)0x77); - - ((IArchiveModifiable)new NrgFormatDescriptor()).Add(ms, [ - ArchiveInputInfo.InMemory(NrgInPlaceModifier.FormatSectorEntryName(30), payload), - ]); - - var grown = ms.ToArray(); - Assert.That(grown.AsSpan(0, 20 * RawSectorSize).ToArray(), - Is.EqualTo(original.AsSpan(0, 20 * RawSectorSize).ToArray())); - - var userDataAt = 30 * RawSectorSize + Mode1DataOffset; - Assert.That(grown.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); - - Assert.That(grown.AsSpan(grown.Length - Ner5FooterSize, Ner5FooterSize).ToArray(), - Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize, Ner5FooterSize).ToArray())); - } - - [Test, Category("RoundTrip")] - public void Descriptor_Remove_ZerosUserData_LeavesEverythingElseIdentical() { - var image = BuildRawMode1Nrg(); - var original = (byte[])image.Clone(); - using var ms = new MemoryStream(image, writable: true); - - ((IArchiveModifiable)new NrgFormatDescriptor()).Remove(ms, [ - NrgInPlaceModifier.FormatSectorEntryName(20), - ]); - - var userDataAt = 20 * RawSectorSize + Mode1DataOffset; - Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), - Is.EqualTo(new byte[Iso9660SectorSize])); - - Assert.That(image.AsSpan(20 * RawSectorSize, 16).ToArray(), - Is.EqualTo(original.AsSpan(20 * RawSectorSize, 16).ToArray())); - - var sectorCount = (image.Length - Ner5FooterSize) / RawSectorSize; - for (var lba = 0; lba < sectorCount; lba++) { - if (lba == 20) continue; - var off = lba * RawSectorSize; - Assert.That(image.AsSpan(off, RawSectorSize).ToArray(), - Is.EqualTo(original.AsSpan(off, RawSectorSize).ToArray()), - $"LBA {lba} unexpectedly mutated."); - } - } - - [Test, Category("RoundTrip")] - public void Descriptor_Replace_RewritesUserDataPreservingFramingAndNeighbours() { - var image = BuildRawMode1Nrg(); - var original = (byte[])image.Clone(); - using var ms = new MemoryStream(image, writable: true); - - var v1 = new byte[Iso9660SectorSize]; - Array.Fill(v1, (byte)0x11); - var v2 = new byte[Iso9660SectorSize]; - Array.Fill(v2, (byte)0x22); - - var modifier = (IArchiveModifiable)new NrgFormatDescriptor(); - modifier.Add(ms, [ArchiveInputInfo.InMemory(NrgInPlaceModifier.FormatSectorEntryName(21), v1)]); - modifier.Add(ms, [ArchiveInputInfo.InMemory(NrgInPlaceModifier.FormatSectorEntryName(21), v2)]); - - var userDataAt = 21 * RawSectorSize + Mode1DataOffset; - Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(v2)); - - Assert.That(image.AsSpan(21 * RawSectorSize, 16).ToArray(), - Is.EqualTo(original.AsSpan(21 * RawSectorSize, 16).ToArray())); - - Assert.That(image.AsSpan(20 * RawSectorSize, RawSectorSize).ToArray(), - Is.EqualTo(original.AsSpan(20 * RawSectorSize, RawSectorSize).ToArray())); - Assert.That(image.AsSpan(22 * RawSectorSize, RawSectorSize).ToArray(), - Is.EqualTo(original.AsSpan(22 * RawSectorSize, RawSectorSize).ToArray())); - } - - [Test, Category("RoundTrip")] - public void MutateThenExtract_RoundTrip() { - var image = BuildRawMode1Nrg(); - using var ms = new MemoryStream(); - ms.Write(image); - - var payload = new byte[Iso9660SectorSize]; - for (var i = 0; i < payload.Length; i++) payload[i] = (byte)(0xAA ^ (i & 0xFF)); - - NrgInPlaceModifier.WriteSector(ms, lba: 25, payload); - - ms.Position = 25L * RawSectorSize + Mode1DataOffset; - var extract = new byte[Iso9660SectorSize]; - var got = 0; - while (got < extract.Length) { - var r = ms.Read(extract, got, extract.Length - got); - if (r == 0) break; - got += r; - } - Assert.That(extract, Is.EqualTo(payload)); - } - - // ── Boundary / contract ──────────────────────────────────────────── - - [Test, Category("Boundary")] - public void WriteSector_WrongUserDataLength_Throws() { - using var ms = new MemoryStream(BuildRawMode1Nrg(), writable: true); - Assert.Throws(() => - NrgInPlaceModifier.WriteSector(ms, 20, new byte[100])); - } - - [Test, Category("Boundary")] - public void WriteSector_NegativeLba_Throws() { - using var ms = new MemoryStream(BuildRawMode1Nrg(), writable: true); - Assert.Throws(() => - NrgInPlaceModifier.WriteSector(ms, -1, new byte[Iso9660SectorSize])); - } - - [Test, Category("HappyPath")] - public void Descriptor_AdvertisesCanModify() { - var d = new NrgFormatDescriptor(); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); - } - - [Test, Category("HappyPath")] - public void Descriptor_ImplementsIArchiveModifiable() { - Assert.That(new NrgFormatDescriptor(), Is.InstanceOf()); - } - - [Test, Category("HappyPath")] - public void TryParseSectorEntryName_RoundTrips() { - var name = NrgInPlaceModifier.FormatSectorEntryName(12345); + [Test] + public void SectorEntryName_RoundTrips() { + var name = NrgInPlaceModifier.FormatSectorEntryName(123); Assert.That(NrgInPlaceModifier.TryParseSectorEntryName(name, out var lba), Is.True); - Assert.That(lba, Is.EqualTo(12345)); - } - - [Test, Category("Boundary")] - public void TryParseSectorEntryName_BogusNames_Rejected() { - Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("readme.txt", out _), Is.False); - Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("sector-abc.bin", out _), Is.False); - Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("sector-.bin", out _), Is.False); - Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("", out _), Is.False); - } - - [Test, Category("Boundary")] - public void Descriptor_Add_NonSectorEntry_IsRefusedAndPreservesImage() { - // This used to assert the image was untouched, and it was -- because the - // entry was dropped on the floor. Leaving the image alone is right; doing it - // without a word is not, and it made the shared "a fresh volume takes one - // more file" check pass on a volume that had taken nothing. - var image = BuildRawMode1Nrg(); - var original = (byte[])image.Clone(); - using var ms = new MemoryStream(image, writable: true); - - Assert.Throws(() => - ((IArchiveModifiable)new NrgFormatDescriptor()).Add(ms, [ - ArchiveInputInfo.InMemory("readme.txt", "hello"u8.ToArray()), - ])); - - Assert.That(image, Is.EqualTo(original), "a refused add must leave the image as it was"); - } - - [Test, Category("Boundary")] - public void DetectGeometry_StripsNer5Footer_FromDataAreaLength() { - var image = BuildRawMode1Nrg(sectorCount: 32); - using var ms = new MemoryStream(image, writable: true); - var geom = NrgInPlaceModifier.DetectGeometry(ms); - Assert.That(geom.SectorSize, Is.EqualTo(RawSectorSize)); - Assert.That(geom.DataOffset, Is.EqualTo(Mode1DataOffset)); - Assert.That(geom.DataAreaLength, Is.EqualTo(32 * RawSectorSize)); + Assert.That(lba, Is.EqualTo(123)); + Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("FILE.BIN", out _), Is.False); } } From f7147b13510cefd5157b1feb2f05e0024c4d3ef8 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:29 +0200 Subject: [PATCH 057/165] docs(onefs): record raw-media feasibility findings --- FileSystems/FileSystem.OneFs/FORMAT-NOTES.md | 85 ++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 FileSystems/FileSystem.OneFs/FORMAT-NOTES.md diff --git a/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md b/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md new file mode 100644 index 000000000..3a5de17a5 --- /dev/null +++ b/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md @@ -0,0 +1,85 @@ +# Dell PowerScale / Isilon OneFS raw-media notes + +## Scope + +`FileSystem.OneFs` is an **offline raw-image** handler. It is not a client for a +running PowerScale cluster and must not silently substitute OneFS REST, SMB, NFS +or SSH operations for the repository's `Stream`-based image contracts. + +The implementation review in September 2026 found enough public information to +model the architecture, but not enough to reconstruct or mutate a standalone +OneFS drive image safely. + +## Publicly documented facts + +Dell's published material establishes the following relevant invariants: + +- OneFS presents a **single global namespace across the cluster**. +- Files and directories are identified by LINs (logical inode numbers). +- The LIN B+ tree maps LINs to **mirrored inode disk addresses**. Dell's own + examples identify a node, drive number, physical block address and inode size. +- File IFM/metatrees map logical block numbers to **protection groups**; directory + DFM trees hold names and namespace relationships. +- Inodes and B+ tree blocks are mirrored; the LIN root is especially heavily + mirrored. +- File data is divided into 8 KiB filesystem blocks and distributed across the + cluster according to protection/layout policy. +- Dell describes OneFS as UFS-based, but the data filesystem is distributed + across nodes. UFS ancestry is therefore not evidence that a single `/ifs` + drive can be parsed or rewritten as a normal standalone UFS volume. + +## What could not be established + +No Dell publication or independently verifiable implementation found during the +review defined enough of the raw-media serialization to support safe offline +mutation. In particular, the review did **not** establish: + +- a fixed offset-zero `"OneFS"` or `"ONEF"` drive signature; +- the complete superblock/LIN-master serialization and version rules; +- inode and B+ tree record encodings across supported OneFS generations; +- free-space/allocation metadata sufficient to distinguish proven-dead blocks; +- protection-group membership and parity/update rules sufficient to relocate or + replace file data; +- journal/transaction ordering required to leave an edited cluster consistent; +- an offline single-drive checker that can act as an independent write oracle. + +The previous descriptor's `"OneFS"` / `"ONEF"` magic therefore had no acceptable +reference and has been removed rather than promoted into more code. + +## Capability decision + +| Capability | State | Reason | +| --- | --- | --- | +| list / extract | limited | synthetic metadata + byte-exact raw-image stream only | +| create | blocked | no public raw-media writer specification/oracle | +| modify / R/W | blocked | namespace, allocation and protection state are cluster-wide | +| purge | blocked | cannot remove live namespace entries and update protection metadata safely | +| wipe | blocked | cannot prove which omitted/raw blocks are unused; guessing is destructive | +| defrag | blocked | relocation requires allocation + IFM/protection-group updates | +| shrink | blocked | no proven tail/free-space ownership or canonical smaller geometry | +| layout | blocked | no proven creator/rebuilder for alternate geometry | +| compact | blocked | composite of defrag/layout/shrink, all blocked above | + +This is deliberately stricter than implementing a no-op maintenance interface: +the support matrix treats an advertised interface as a working verb. Reporting a +successful wipe that merely writes zero bytes would be technically callable but +operationally dishonest. + +## References + +- Dell Technologies Info Hub, **OneFS Metadata**: + https://infohub.delltechnologies.com/en-us/p/onefs-metadata/ +- Dell PowerScale OneFS Technical Overview, **File system structure**: + https://infohub.delltechnologies.com/en-nz/l/dell-powerscale-onefs-technical-overview/file-system-structure/1/ +- Dell PowerScale OneFS 9.15 Administration Guide, **Structure of the file system**: + https://www.dell.com/support/manuals/en-us/isilon-onefs/ifs_pub_91500_administration_guide_gui/structure-of-the-file-system +- Dell PowerScale OneFS Technical Specifications Guide, **File system guidelines** + (8 KiB block size): + https://www.dell.com/support/manuals/en-us/isilon-onefs/ifs_pub_onefs_tech_spec_guide_9.9.0.0/file-system-guidelines + +## Licensing / implementation provenance + +The sources above are used only for factual behavior and architecture. No Dell +implementation source, comments, naming, control flow or other expressive code +was copied or translated. The implementation is an original clean-room C# +inspection surface under this repository's license. From 359d8128af20d784d53403220ee0dd45e4402527 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:53 +0200 Subject: [PATCH 058/165] Fix MooseFS metadata envelope parsing --- .../FileSystem.MooseFs/MooseFsReader.cs | 379 +++++++++--------- 1 file changed, 188 insertions(+), 191 deletions(-) diff --git a/FileSystems/FileSystem.MooseFs/MooseFsReader.cs b/FileSystems/FileSystem.MooseFs/MooseFsReader.cs index 14245eebc..f3532f772 100644 --- a/FileSystems/FileSystem.MooseFs/MooseFsReader.cs +++ b/FileSystems/FileSystem.MooseFs/MooseFsReader.cs @@ -6,110 +6,78 @@ namespace FileSystem.MooseFs; /// -/// Partial R/O reader for MooseFS master-metadata images (metadata.mfs). -/// MooseFS is a fault-tolerant distributed FS — the master server keeps the -/// namespace + chunk-server topology in a single binary metadata file, while -/// file data lives on chunk servers. This reader understands the master -/// metadata's outer envelope: -/// -/// 8-byte ASCII signature (e.g. MFSM 2.0, MFSM 1.6, -/// MFSM 1.5, MFSM 1.4, MFSM NEW). -/// For 1.6+ images: two 8-byte big-endian counters -/// (file-id counter, metadata version) immediately after the signature. -/// Sequence of sections. Each section: 8-byte ASCII type tag -/// (SESS 1.0, STAT 1.0, NODE 1.0, EDGE 1.0, -/// FREE 1.0, XATR 1.0, CHNK 1.0, OPEN 1.0, -/// FLCK 1.0, QUOT 1.0, ACLS 1.0, …) + 8-byte -/// big-endian payload length + that many payload bytes. -/// Final 16-byte terminator [MFS EOF MARKER]. -/// -/// -/// -/// The reader walks the section index only — it does not attempt to -/// decode NODE / EDGE record bodies, which differ between MooseFS minor -/// versions and require ground-truth golden samples to validate. NODE/EDGE -/// would give path tree + inode metadata; CHNK gives chunk-id mappings. None -/// of those by themselves yield file content — MooseFS data lives on chunk -/// servers and is only reachable via the live MooseFS protocol. Therefore -/// the reader exposes: -/// -/// -/// metadata.ini — human-readable summary of header + section -/// table (name, payload offset, payload length). -/// moosefs-master.bin — the raw image, byte-for-byte. -/// section_<NAME>.bin — the raw payload bytes of each -/// section the index walk surfaced (NODE, EDGE, CHNK, …). Useful for -/// offline forensics; we make no claim about their internal -/// structure. -/// -/// -/// -/// If section-walk fails (signature past the 8-byte tag is not recognised, -/// a section length runs past EOF, the EOF marker is missing, …), the -/// reader falls back to a header-only surface (metadata.ini + raw) and -/// records the parse failure in metadata.ini's parse_status -/// field. This is the honest "we recognise the envelope but couldn't walk -/// the contents" mode rather than silently inventing entries. -/// +/// Partial reader for MooseFS master-metadata images (metadata.mfs). +/// It understands the versioned outer envelope and section framing, but does +/// not decode the version-specific NODE / EDGE / CHNK bodies or contact chunk +/// servers. Consequently all surfaced archive entries are synthetic forensic +/// views of the metadata image rather than the mounted MooseFS namespace. /// public sealed class MooseFsReader : IDisposable { - /// MooseFS master metadata 4-byte prefix: ASCII "MFSM". + /// MooseFS master metadata 4-byte prefix: ASCII MFSM. public static readonly byte[] MasterTag = "MFSM"u8.ToArray(); - /// 16-byte MooseFS end-of-file marker following the last section. + /// Modern (1.6+) 16-byte MooseFS end-of-file marker. public static readonly byte[] EofMarker = "[MFS EOF MARKER]"u8.ToArray(); - private const int HeaderSize = 8; - // After the 8-byte signature, modern (1.6+) images have two BE uint64s - // followed by the section stream. Section tag + length = 16 bytes. - private const int SectionTagSize = 8; - private const int SectionLenSize = 8; - // Section payloads can be huge on a real cluster — cap what we'll hold in - // memory as a synthetic per-section entry. Larger sections still show up - // in metadata.ini's table but their raw payload is not surfaced. + private const int SignatureSize = 8; + private const int MetadataHeaderSize = 16; + private const int SectionHeaderSize = 16; private const long MaxInMemorySection = 64L * 1024 * 1024; private readonly byte[] _data; private readonly List _entries = []; private readonly List _sections = []; - /// Listing of every entry this image surfaces. + /// Listing of every synthetic entry surfaced from this image. public IReadOnlyList Entries => _entries; - /// Section index walked from the master metadata stream. + /// Section index walked from section-framed (1.6+) metadata. public IReadOnlyList Sections => _sections; - /// The 8-byte ASCII signature at offset 0 (e.g. "MFSM 2.0"). + /// The 8-byte ASCII signature, for example MFSM 2.0. public string Signature { get; private set; } = ""; - /// True when the 4-byte MFSM tag was present at offset 0. + /// True once the MFSM prefix has been verified. public bool ValidHeader { get; private set; } /// - /// File-id counter from the modern (1.6+) post-signature header, or - /// null when the image is too short or pre-1.6. + /// Packed file-format version (0x16 for 1.6, 0x20 for 2.0), + /// or null for the special MFSM NEW bootstrap image. /// - public ulong? FileIdCounter { get; private set; } + public byte? FileFormatVersion { get; private set; } + + /// Whether this is MooseFS's official eight-byte empty bootstrap image. + public bool IsEmptyBootstrap { get; private set; } /// - /// Metadata version counter from the modern (1.6+) post-signature header, - /// or null when the image is too short or pre-1.6. + /// Maximum node id from the pre-2.0 metadata header. Not present in 2.0+. /// + public uint? MaxNodeId { get; private set; } + + /// Metadata/changelog version stored in the master metadata header. public ulong? MetadataVersion { get; private set; } /// - /// Human-readable description of how the section walk terminated: - /// "ok" (full walk + EOF marker), "truncated" (section walk - /// stopped before EOF marker), "header-only" (image too short for - /// any sections), or "unsupported-header" (no MFSM tag). + /// Next session id from the pre-2.0 metadata header. Not present in 2.0+. /// - public string ParseStatus { get; private set; } = "unsupported-header"; + public uint? NextSessionId { get; private set; } + + /// Metadata instance id stored by 2.0+ images. + public ulong? MetaId { get; private set; } + + /// The exact image size consumed by this reader. + public long ImageSize => _data.LongLength; /// - /// Initializes a new instance of . + /// Human-readable parse result: ok, header-only, + /// truncated, trailing-data, or unsupported-header. /// + public string ParseStatus { get; private set; } = "unsupported-header"; + + /// Initializes a reader over one MooseFS metadata image. public MooseFsReader(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); using var ms = new MemoryStream(); stream.CopyTo(ms); _data = ms.ToArray(); @@ -117,145 +85,169 @@ public MooseFsReader(Stream stream) { } private void Parse() { - if (_data.Length < HeaderSize) - throw new InvalidDataException("MooseFS: file too small for master metadata header."); + if (_data.Length < SignatureSize) + throw new InvalidDataException("MooseFS: file too small for master metadata signature."); - if (!_data.AsSpan(0, 4).SequenceEqual(MasterTag)) + if (!_data.AsSpan(0, MasterTag.Length).SequenceEqual(MasterTag)) throw new InvalidDataException("MooseFS: missing 'MFSM' tag at offset 0."); this.ValidHeader = true; - this.Signature = Encoding.ASCII.GetString(_data, 0, 8).TrimEnd('\0'); - - // Modern (1.6+) images carry two BE uint64s right after the 8-byte - // signature. Older 1.4/1.5 images stream sections immediately. We pull - // the counters when the image is long enough; downstream code does not - // assume they're meaningful for non-1.6+ signatures. - if (_data.Length >= HeaderSize + 16) { - this.FileIdCounter = BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(HeaderSize, 8)); - this.MetadataVersion = BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(HeaderSize + 8, 8)); + this.Signature = Encoding.ASCII.GetString(_data, 0, SignatureSize); + + if (this.Signature == "MFSM NEW") { + this.IsEmptyBootstrap = true; + this.ParseStatus = _data.Length == SignatureSize ? "ok" : "trailing-data"; + BuildEntries(); + return; } - WalkSections(); + if (!TryParseFileFormatVersion(_data.AsSpan(0, SignatureSize), out var fileVersion)) { + this.ParseStatus = "unsupported-header"; + BuildEntries(); + return; + } - var meta = BuildMetadata(); - _entries.Add(new MooseFsEntry { - Name = "metadata.ini", - Size = meta.Length, - IsDirectory = false, - Offset = 0, - Data = meta, - }); - _entries.Add(new MooseFsEntry { - Name = "moosefs-master.bin", - Size = _data.Length, - IsDirectory = false, - Offset = 0, - Data = _data, - }); + this.FileFormatVersion = fileVersion; + if (_data.Length < SignatureSize + MetadataHeaderSize) { + this.ParseStatus = "header-only"; + BuildEntries(); + return; + } - foreach (var s in _sections) { - // Bound the per-section payload we materialise so a 4-GB CHNK section - // does not allocate 4 GB of synthetic-entry buffer. The section still - // appears in metadata.ini's table — we just refuse to mirror the - // payload as a separate entry. The full raw image stays accessible - // via moosefs-master.bin. - if (s.Length <= 0 || s.Length > MaxInMemorySection) - continue; - // Defensive: the walker already bounds Offset+Length to _data.Length, - // but re-check before slicing to guard against any future refactor. - if (s.Offset < 0 || s.Offset + s.Length > _data.Length) - continue; - var payload = _data.AsSpan((int)s.Offset, (int)s.Length).ToArray(); - _entries.Add(new MooseFsEntry { - Name = $"section_{SanitiseSectionName(s.Tag)}.bin", - Size = payload.Length, - IsDirectory = false, - Offset = s.Offset, - Data = payload, - }); + ParseMetadataHeader(fileVersion); + + if (fileVersion < 0x16) + ParseLegacyBody(); + else + WalkSections(); + + BuildEntries(); + } + + private void ParseMetadataHeader(byte fileVersion) { + var header = _data.AsSpan(SignatureSize, MetadataHeaderSize); + if (fileVersion >= 0x20) { + this.MetadataVersion = BinaryPrimitives.ReadUInt64BigEndian(header[..8]); + this.MetaId = BinaryPrimitives.ReadUInt64BigEndian(header[8..]); + return; } + + this.MaxNodeId = BinaryPrimitives.ReadUInt32BigEndian(header[..4]); + this.MetadataVersion = BinaryPrimitives.ReadUInt64BigEndian(header.Slice(4, 8)); + this.NextSessionId = BinaryPrimitives.ReadUInt32BigEndian(header[12..]); } - private void WalkSections() { - // Section stream begins after signature for 1.4/1.5, after signature + 16 - // counter bytes for 1.6+. We pick the offset by checking signature - // version: anything other than the 1.4/1.5 strings gets the modern - // offset. This still correctly rejects malformed images because the - // first section tag must be all-ASCII printable. - var sectionStart = HeaderSize; - if (this.Signature is not ("MFSM 1.4" or "MFSM 1.5") && _data.Length >= HeaderSize + 16) - sectionStart = HeaderSize + 16; - - if (_data.Length < sectionStart + SectionTagSize + SectionLenSize) { - this.ParseStatus = "header-only"; + private void ParseLegacyBody() { + // Before metadata format 1.6 MooseFS did not wrap NODE/EDGE/etc. in the + // modern 16-byte section envelope. The authoritative checker only relies + // on the final 16 zero bytes, so preserve the whole legacy body as opaque + // metadata instead of guessing record boundaries. + var minimumLength = SignatureSize + MetadataHeaderSize + 16; + if (_data.Length < minimumLength) { + this.ParseStatus = "truncated"; return; } - var offset = sectionStart; - while (offset + SectionTagSize + SectionLenSize <= _data.Length) { - // EOF marker is 16 bytes [MFS EOF MARKER] and is NOT followed by a - // length field — once we see it, the walk is complete. - if (offset + EofMarker.Length <= _data.Length - && _data.AsSpan(offset, EofMarker.Length).SequenceEqual(EofMarker)) { - this.ParseStatus = "ok"; + this.ParseStatus = _data.AsSpan(_data.Length - 16, 16).IndexOfAnyExcept((byte)0) < 0 + ? "ok" + : "truncated"; + } + + private void WalkSections() { + var offset = SignatureSize + MetadataHeaderSize; + while (offset + SectionHeaderSize <= _data.Length) { + if (_data.AsSpan(offset, EofMarker.Length).SequenceEqual(EofMarker)) { + this.ParseStatus = offset + EofMarker.Length == _data.Length ? "ok" : "trailing-data"; return; } - var tagBytes = _data.AsSpan(offset, SectionTagSize); + var tagBytes = _data.AsSpan(offset, 8); if (!IsPlausibleSectionTag(tagBytes)) { - // Section walk derailed. Keep what we have, mark truncated. We do - // not throw — header-level surfaces remain useful even if section - // walk failed (older / future MooseFS versions, partially-written - // images, dump tools that strip framing, …). this.ParseStatus = "truncated"; return; } - var tag = Encoding.ASCII.GetString(tagBytes).TrimEnd(); - var lenU64 = BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(offset + SectionTagSize, SectionLenSize)); - // A single section larger than long.MaxValue would mean the image is - // pathological or this isn't really a section header — treat as walk - // failure rather than overflowing int math. - if (lenU64 > long.MaxValue) { + + var length64 = BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(offset + 8, 8)); + if (length64 > long.MaxValue) { this.ParseStatus = "truncated"; return; } - var len = (long)lenU64; - var payloadOffset = (long)offset + SectionTagSize + SectionLenSize; - if (payloadOffset + len > _data.Length) { - // Section claims more bytes than the image has — truncated dump or - // bogus length. Record what we got and stop. + + var length = (long)length64; + var payloadOffset = (long)offset + SectionHeaderSize; + if (length > _data.LongLength - payloadOffset) { this.ParseStatus = "truncated"; return; } - _sections.Add(new SectionEntry(tag, payloadOffset, len)); - offset = (int)(payloadOffset + len); + var tag = Encoding.ASCII.GetString(tagBytes); + _sections.Add(new SectionEntry(tag, payloadOffset, length)); + offset = checked((int)(payloadOffset + length)); } - // Walked past the last possible section header but never saw the EOF - // marker — image is truncated. The accumulated sections are still real. this.ParseStatus = "truncated"; } - private static bool IsPlausibleSectionTag(ReadOnlySpan tagBytes) { - // A valid MooseFS section tag is 8 ASCII bytes: 4-char family - // ("SESS", "NODE", "EDGE", "CHNK", "FREE", "XATR", "STAT", "OPEN", - // "FLCK", "QUOT", "ACLS", …), a space, then a version string like - // "1.0" or "2.0". We sanity-check that every byte is printable ASCII — - // this rejects random binary that just happens to be at the right - // offset without locking us to a fixed family allow-list (MooseFS - // adds new section types between minor versions). - foreach (var b in tagBytes) - if (b is < 0x20 or > 0x7E) - return false; + private void BuildEntries() { + var metadata = BuildMetadata(); + _entries.Add(new MooseFsEntry { + Name = "metadata.ini", + Size = metadata.Length, + IsDirectory = false, + Offset = 0, + Data = metadata, + }); + _entries.Add(new MooseFsEntry { + Name = "moosefs-master.bin", + Size = _data.Length, + IsDirectory = false, + Offset = 0, + Data = _data, + }); + + foreach (var section in _sections) { + if (section.Length <= 0 || section.Length > MaxInMemorySection) + continue; + if (section.Offset < 0 || section.Offset + section.Length > _data.LongLength) + continue; + + var payload = _data.AsSpan((int)section.Offset, (int)section.Length).ToArray(); + _entries.Add(new MooseFsEntry { + Name = $"section_{SanitiseSectionName(section.Tag)}.bin", + Size = payload.Length, + IsDirectory = false, + Offset = section.Offset, + Data = payload, + }); + } + } + + private static bool TryParseFileFormatVersion(ReadOnlySpan signature, out byte version) { + version = 0; + if (signature.Length != SignatureSize + || !signature[..5].SequenceEqual("MFSM "u8) + || signature[5] is < (byte)'1' or > (byte)'9' + || signature[6] != (byte)'.' + || signature[7] is < (byte)'0' or > (byte)'9') + return false; + + version = (byte)(((signature[5] - (byte)'0') << 4) | (signature[7] - (byte)'0')); return true; } + private static bool IsPlausibleSectionTag(ReadOnlySpan tag) { + if (tag.Length != 8 || tag[4] != (byte)' ' || tag[6] != (byte)'.') + return false; + + for (var i = 0; i < 4; ++i) + if (tag[i] is not (>= (byte)'A' and <= (byte)'Z') and not (>= (byte)'0' and <= (byte)'9')) + return false; + + return tag[5] is >= (byte)'0' and <= (byte)'9' + && tag[7] is >= (byte)'0' and <= (byte)'9'; + } + private static string SanitiseSectionName(string tag) { - // Section tags include a space ("NODE 1.0") which makes for awkward - // file names. Collapse whitespace runs to a single underscore and - // drop any non-alphanumeric tail so the synthetic entry names stay - // POSIX-portable and round-trip through Extract → WriteFile. var sb = new StringBuilder(tag.Length); foreach (var c in tag) { if (char.IsLetterOrDigit(c)) @@ -263,8 +255,8 @@ private static string SanitiseSectionName(string tag) { else if (sb.Length > 0 && sb[^1] != '_') sb.Append('_'); } - var s = sb.ToString().TrimEnd('_'); - return s.Length == 0 ? "unnamed" : s; + var result = sb.ToString().TrimEnd('_'); + return result.Length == 0 ? "unnamed" : result; } private byte[] BuildMetadata() { @@ -275,37 +267,42 @@ private byte[] BuildMetadata() { bldr.Append("magic_tag=MFSM\n"); bldr.Append("magic_offset=0\n"); bldr.Append(CultureInfo.InvariantCulture, $"image_size={_data.Length}\n"); - if (this.FileIdCounter.HasValue) - bldr.Append(CultureInfo.InvariantCulture, $"file_id_counter={this.FileIdCounter.Value}\n"); + if (this.IsEmptyBootstrap) + bldr.Append("empty_bootstrap=true\n"); + if (this.FileFormatVersion is byte version) + bldr.Append(CultureInfo.InvariantCulture, + $"file_format_version={version >> 4}.{version & 0x0F}\n"); + if (this.MaxNodeId.HasValue) + bldr.Append(CultureInfo.InvariantCulture, $"max_node_id={this.MaxNodeId.Value}\n"); if (this.MetadataVersion.HasValue) bldr.Append(CultureInfo.InvariantCulture, $"metadata_version={this.MetadataVersion.Value}\n"); + if (this.NextSessionId.HasValue) + bldr.Append(CultureInfo.InvariantCulture, $"next_session_id={this.NextSessionId.Value}\n"); + if (this.MetaId.HasValue) + bldr.Append(CultureInfo.InvariantCulture, $"meta_id={this.MetaId.Value}\n"); bldr.Append(CultureInfo.InvariantCulture, $"section_count={_sections.Count}\n"); - for (var i = 0; i < _sections.Count; i++) { - var s = _sections[i]; + for (var i = 0; i < _sections.Count; ++i) { + var section = _sections[i]; bldr.Append(CultureInfo.InvariantCulture, - $"section[{i}]={s.Tag} offset={s.Offset} length={s.Length}\n"); + $"section[{i}]={section.Tag} offset={section.Offset} length={section.Length}\n"); } - bldr.Append("note=Partial R/O — outer envelope (signature + section index) "); - bldr.Append("only. NODE/EDGE/CHNK body decoding is version-specific and "); - bldr.Append("requires golden samples to validate honestly. File content "); - bldr.Append("lives on chunk servers and is not reachable from this image.\n"); + bldr.Append("note=Partial metadata-image view only. NODE/EDGE/CHNK body decoding is version-specific; "); + bldr.Append("file content lives on chunk servers and is not reachable from metadata.mfs alone.\n"); return Encoding.UTF8.GetBytes(bldr.ToString()); } - /// Returns the bytes that back the given entry (in-memory). + /// Returns the bytes backing a surfaced synthetic entry. public byte[] Extract(MooseFsEntry entry) { ArgumentNullException.ThrowIfNull(entry); return entry.Data; } - /// - /// Releases resources held by this instance. - /// + /// Releases resources held by this instance. public void Dispose() { } - /// One walked section from the master metadata stream. - /// The 8-byte ASCII tag (e.g. "NODE 1.0"), trimmed. - /// Byte offset of the section payload (after the 16-byte tag+length). - /// Length of the section payload in bytes. + /// One walked section from a 1.6+ master metadata stream. + /// Eight-byte section tag, for example NODE 1.0. + /// Byte offset of the section payload. + /// Payload length in bytes. public readonly record struct SectionEntry(string Tag, long Offset, long Length); } From fe7cc103117a251db4ed2d7eb53bbf8f55249001 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:54 +0200 Subject: [PATCH 059/165] Cover T64 compatibility and maintenance operations --- Compression.Tests/T64/T64MaintenanceTests.cs | 206 +++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 Compression.Tests/T64/T64MaintenanceTests.cs diff --git a/Compression.Tests/T64/T64MaintenanceTests.cs b/Compression.Tests/T64/T64MaintenanceTests.cs new file mode 100644 index 000000000..4661f0eb5 --- /dev/null +++ b/Compression.Tests/T64/T64MaintenanceTests.cs @@ -0,0 +1,206 @@ +using System.Buffers.Binary; +using Compression.Registry; +using FileFormat.T64; + +namespace Compression.Tests.T64; + +[TestFixture] +public class T64MaintenanceTests { + + [Test, Category("Compatibility")] + public void Reader_Conv64BrokenEndAddress_UsesPhysicalPayloadExtent() { + var payload = "HELLO"u8.ToArray(); + var writer = new T64Writer(); + writer.AddFile("BROKEN", 0x0801, payload); + var image = writer.Build(); + + BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(64 + 4), 0xC3C6); + + using var stream = new MemoryStream(image); + using var reader = new T64Reader(stream); + Assert.That(reader.Entries, Has.Count.EqualTo(1)); + Assert.That(reader.Entries[0].Size, Is.EqualTo(payload.Length)); + Assert.That(reader.Extract(reader.Entries[0]), Is.EqualTo(payload)); + } + + [Test, Category("EdgeCase")] + public void Writer_EntryEndingAt10000_RoundTripsFullPayload() { + var payload = Enumerable.Range(0, 0x4000).Select(static i => (byte)i).ToArray(); + var writer = new T64Writer(); + writer.AddFile("TOPMEM", 0xC000, payload); + + using var stream = new MemoryStream(writer.Build()); + using var reader = new T64Reader(stream); + var entry = reader.Entries.Single(); + + Assert.That(entry.EndAddress, Is.Zero, "exclusive $10000 end must wrap in the 16-bit field"); + Assert.That(entry.Size, Is.EqualTo(payload.Length)); + Assert.That(reader.Extract(entry), Is.EqualTo(payload)); + } + + [Test, Category("ErrorHandling")] + public void Reader_DirectoryPastEof_Throws() { + var image = new byte[64]; + "C64S tape image file"u8.CopyTo(image); + BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(32), 0x0100); + BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(34), 1); + + using var stream = new MemoryStream(image); + Assert.Throws(() => _ = new T64Reader(stream)); + } + + [Test, Category("ErrorHandling")] + public void Add_InvalidReplacement_DoesNotDeleteExistingEntry() { + var oldPayload = "keep-me"u8.ToArray(); + var writer = new T64Writer(); + writer.AddFile("TARGET", oldPayload); + using var stream = Writable(writer.Build()); + + Assert.Throws(() => + T64InPlaceModifier.AddFile(stream, "TARGET", new byte[0x10000], 0x0801)); + + using var reader = new T64Reader(stream); + Assert.That(reader.Entries.Select(static e => e.Name), Is.EqualTo(new[] { "TARGET" })); + Assert.That(reader.Extract(reader.Entries.Single()), Is.EqualTo(oldPayload)); + } + + [Test, Category("RoundTrip")] + public void BlockMover_ForwardOverlap_IsMemmoveSafeAndDoesNotZeroDestination() { + using var stream = Writable([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + + new T64BlockMover().MoveExtent(stream, 0, 2, 6, zeroSource: true); + + Assert.That(stream.ToArray(), Is.EqualTo(new byte[] { 0, 0, 1, 2, 3, 4, 5, 6, 9, 10 })); + } + + [Test, Category("RoundTrip")] + public void Defragment_MovesPayloadToDirectoryEnd_AndPreservesMetadata() { + var (stream, payload) = BuildGappedImage(); + using (stream) { + var descriptor = new T64FormatDescriptor(); + descriptor.Defragment(stream); + + Assert.That(stream.Length, Is.EqualTo(64 + 2 * 32 + payload.Length)); + using var reader = new T64Reader(stream); + var entry = reader.Entries.Single(); + Assert.Multiple(() => { + Assert.That(entry.DataOffset, Is.EqualTo(128)); + Assert.That(entry.StartAddress, Is.EqualTo(0x2000)); + Assert.That(entry.FileType, Is.EqualTo(0x81)); + Assert.That(reader.Version, Is.EqualTo(0x0101)); + Assert.That(reader.TapeName, Is.EqualTo("GAPPED TAPE")); + Assert.That(reader.Extract(entry), Is.EqualTo(payload)); + }); + } + } + + [Test, Category("RoundTrip")] + public void WipeUnusedSpace_ZerosFreeDirectorySlotAndPayloadGaps() { + var (stream, payload) = BuildGappedImage(); + using (stream) { + var descriptor = new T64FormatDescriptor(); + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(stream); + var raw = stream.ToArray(); + + Assert.Multiple(() => { + Assert.That(wiped, Is.EqualTo(32 + 4 + 3)); + Assert.That(raw.AsSpan(96, 32).ToArray(), Is.All.Zero); + Assert.That(raw.AsSpan(128, 4).ToArray(), Is.All.Zero); + Assert.That(raw.AsSpan(132, payload.Length).ToArray(), Is.EqualTo(payload)); + Assert.That(raw.AsSpan(132 + payload.Length, 3).ToArray(), Is.All.Zero); + }); + } + } + + [Test, Category("RoundTrip")] + public void Shrink_TightPacksAndPreservesT64Metadata() { + var (input, payload) = BuildGappedImage(); + using (input) + using (var output = new MemoryStream()) { + var descriptor = new T64FormatDescriptor(); + ((IArchiveShrinkable)descriptor).Shrink(input, output); + + Assert.That(output.Length, Is.EqualTo(64 + 32 + payload.Length)); + using var reader = new T64Reader(output); + var entry = reader.Entries.Single(); + Assert.Multiple(() => { + Assert.That(reader.DirectoryEntryCount, Is.EqualTo(1)); + Assert.That(reader.Version, Is.EqualTo(0x0101)); + Assert.That(reader.TapeName, Is.EqualTo("GAPPED TAPE")); + Assert.That(entry.StartAddress, Is.EqualTo(0x2000)); + Assert.That(entry.FileType, Is.EqualTo(0x81)); + Assert.That(reader.Extract(entry), Is.EqualTo(payload)); + }); + } + } + + [Test, Category("RoundTrip")] + public void Purge_LeavesValidCanonicalEmptyImage() { + var (stream, _) = BuildGappedImage(); + using (stream) { + var descriptor = new T64FormatDescriptor(); + ((IArchivePurgeable)descriptor).Purge(stream); + + Assert.That(stream.Length, Is.EqualTo(64)); + using var reader = new T64Reader(stream); + Assert.Multiple(() => { + Assert.That(reader.Entries, Is.Empty); + Assert.That(reader.DirectoryEntryCount, Is.Zero); + Assert.That(reader.UsedEntryCount, Is.Zero); + Assert.That(reader.Version, Is.EqualTo(0x0101)); + Assert.That(reader.TapeName, Is.EqualTo("GAPPED TAPE")); + }); + } + } + + [Test, Category("HappyPath")] + public void Descriptor_ExposesSupportedMaintenanceCapabilities() { + var descriptor = new T64FormatDescriptor(); + Assert.Multiple(() => { + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf(), + "T64 has no selectable filesystem/block geometry to re-layout."); + }); + } + + private static (MemoryStream Stream, byte[] Payload) BuildGappedImage() { + var payload = "PAYLOAD"u8.ToArray(); + const int payloadOffset = 132; // 64-byte header + 2 slots + 4 dead bytes + var image = new byte[payloadOffset + 7 + 3]; + + "C64S tape image file"u8.CopyTo(image); + BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(32), 0x0101); + BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(34), 2); + BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(36), 1); + "GAPPED TAPE"u8.CopyTo(image.AsSpan(40)); + image.AsSpan(51, 13).Fill(0x20); + + image[64] = 1; + image[65] = 0x81; + BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(66), 0x2000); + BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(68), 0x2000 + payload.Length); + BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(72), payloadOffset); + "GAPTEST"u8.CopyTo(image.AsSpan(80)); + image.AsSpan(87, 9).Fill(0x20); + + // Free directory slot containing stale metadata, then dead payload gap/tail. + image.AsSpan(96, 32).Fill(0xA5); + image[96] = 0; + image.AsSpan(128, 4).Fill(0xD1); + payload.CopyTo(image, payloadOffset); + image.AsSpan(payloadOffset + payload.Length, 3).Fill(0xD2); + + return (Writable(image), payload); + } + + private static MemoryStream Writable(byte[] bytes) { + var result = new MemoryStream(); + result.Write(bytes); + result.Position = 0; + return result; + } +} From 5764e400b95e8ad99b8e5b7b9c645779283c410c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:58 +0200 Subject: [PATCH 060/165] Test conservative TUX3 maintenance operations --- Compression.Tests/Tux3/Tux3Tests.cs | 114 ++++++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 5 deletions(-) diff --git a/Compression.Tests/Tux3/Tux3Tests.cs b/Compression.Tests/Tux3/Tux3Tests.cs index 58c98b852..448adba33 100644 --- a/Compression.Tests/Tux3/Tux3Tests.cs +++ b/Compression.Tests/Tux3/Tux3Tests.cs @@ -7,15 +7,19 @@ namespace Compression.Tests.Tux3; [TestFixture] public class Tux3Tests { - private static byte[] BuildNativeImage(bool legacy2012 = false) { - var image = new byte[16 * 1024]; + private static byte[] BuildNativeImage( + bool legacy2012 = false, + ulong volBlocks = 0x1234, + ushort blockBits = 12, + int imageLength = 16 * 1024) { + var image = new byte[imageLength]; var super = image.AsSpan(Tux3Reader.SuperblockOffset, Tux3Reader.DiskSuperSize); (legacy2012 ? Tux3Reader.Legacy2012Magic : Tux3Reader.Magic).CopyTo(super); BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x08, 8), 0x0123_4567_89AB_CDEFUL); BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x10, 8), 0x1020_3040_5060_7080UL); - BinaryPrimitives.WriteUInt16BigEndian(super.Slice(0x18, 2), 12); - BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x20, 8), 0x0000_0000_0000_1234UL); + BinaryPrimitives.WriteUInt16BigEndian(super.Slice(0x18, 2), blockBits); + BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x20, 8), volBlocks); BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x28, 8), 0x0001_0000_0000_0042UL); BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x30, 8), 0x8002_0000_0000_0043UL); BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x38, 8), 0x0000_0000_0000_0040UL); @@ -75,7 +79,7 @@ public void FormerPrivateTux3SuprMagic_IsRejected() { } [Test, Category("Spec")] - public void Descriptor_AdvertisesOnlyTheNativeMetadataSurface() { + public void Descriptor_AdvertisesNativeMetadataAndSafeMaintenanceSurface() { var descriptor = new Tux3FormatDescriptor(); Assert.Multiple(() => { @@ -85,8 +89,12 @@ public void Descriptor_AdvertisesOnlyTheNativeMetadataSurface() { Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.SupportsMultipleEntries), Is.False); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); Assert.That(descriptor, Is.Not.InstanceOf()); Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); Assert.That(descriptor.Description, Does.Contain("native big-endian superblock")); }); } @@ -114,4 +122,100 @@ public void Descriptor_ExtractsNativeMetadataWithoutInventedFileTable() { Directory.Delete(output, recursive: true); } } + + [Test, Category("Spec")] + public void ExtentMap_ReservesUndecodedVolumeAndMarksOnlyExternalTailFree() { + const long declaredLength = 3L << 12; + var image = BuildNativeImage(volBlocks: 3); + using var stream = new MemoryStream(image, writable: false); + + var extents = new Tux3FormatDescriptor().EnumerateExtents(stream).ToArray(); + + Assert.That(extents, Has.Length.EqualTo(2)); + Assert.Multiple(() => { + Assert.That(extents[0].Offset, Is.Zero); + Assert.That(extents[0].Length, Is.EqualTo(declaredLength)); + Assert.That(extents[0].Kind, Is.EqualTo(DefragBlockKind.MetadataReserved)); + Assert.That(extents[1].Offset, Is.EqualTo(declaredLength)); + Assert.That(extents[1].Length, Is.EqualTo(image.LongLength - declaredLength)); + Assert.That(extents[1].Kind, Is.EqualTo(DefragBlockKind.Free)); + }); + } + + [Test, Category("HappyPath")] + public void WipeUnusedSpace_ZeroesOnlyExternalTail() { + const int declaredLength = 3 << 12; + var image = BuildNativeImage(volBlocks: 3); + image[declaredLength - 1] = 0x5A; + Array.Fill(image, (byte)0xA5, declaredLength, image.Length - declaredLength); + using var stream = new MemoryStream(image, writable: true); + var descriptor = new Tux3FormatDescriptor(); + + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(stream, wipeClusterTips: false, wipeDeletedEntries: false); + + Assert.Multiple(() => { + Assert.That(wiped, Is.EqualTo(image.Length - declaredLength)); + Assert.That(image[declaredLength - 1], Is.EqualTo(0x5A)); + Assert.That(image.AsSpan(declaredLength).ToArray(), Is.All.Zero); + }); + } + + [Test, Category("HappyPath")] + public void Shrink_RemovesOnlyExternalTail() { + const int declaredLength = 3 << 12; + var image = BuildNativeImage(volBlocks: 3); + Array.Fill(image, (byte)0xA5, declaredLength, image.Length - declaredLength); + using var input = new MemoryStream(image, writable: false); + using var output = new MemoryStream(); + + new Tux3FormatDescriptor().Shrink(input, output); + + Assert.Multiple(() => { + Assert.That(output.Length, Is.EqualTo(declaredLength)); + Assert.That(output.ToArray(), Is.EqualTo(image[..declaredLength])); + }); + } + + [Test, Category("Regression")] + public void TruncatedDeclaredVolume_IsNeverTreatedAsFreeOrShrunk() { + var image = BuildNativeImage(volBlocks: 5); + image[^1] = 0xA5; + var descriptor = new Tux3FormatDescriptor(); + + using var layoutStream = new MemoryStream(image, writable: false); + var extents = descriptor.EnumerateExtents(layoutStream).ToArray(); + Assert.That(extents, Has.Length.EqualTo(1)); + Assert.Multiple(() => { + Assert.That(extents[0].Offset, Is.Zero); + Assert.That(extents[0].Length, Is.EqualTo(image.LongLength)); + Assert.That(extents[0].Kind, Is.EqualTo(DefragBlockKind.MetadataReserved)); + }); + + var wipeCopy = image.ToArray(); + using var wipeStream = new MemoryStream(wipeCopy, writable: true); + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(wipeStream, wipeClusterTips: false, wipeDeletedEntries: false); + Assert.Multiple(() => { + Assert.That(wiped, Is.Zero); + Assert.That(wipeCopy, Is.EqualTo(image)); + }); + + using var shrinkInput = new MemoryStream(image, writable: false); + using var shrinkOutput = new MemoryStream(); + descriptor.Shrink(shrinkInput, shrinkOutput); + Assert.That(shrinkOutput.ToArray(), Is.EqualTo(image)); + } + + [Test, Category("Regression")] + public void OverflowingDeclaredVolume_FailsClosed() { + var image = BuildNativeImage(volBlocks: ulong.MaxValue, blockBits: 62); + var descriptor = new Tux3FormatDescriptor(); + + using var layoutStream = new MemoryStream(image, writable: false); + Assert.That(descriptor.EnumerateExtents(layoutStream), Is.Empty); + + using var input = new MemoryStream(image, writable: false); + using var output = new MemoryStream(); + descriptor.Shrink(input, output); + Assert.That(output.ToArray(), Is.EqualTo(image)); + } } From 0a7fc6870bbf8c4f84968494d16af496461d1419 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:04:59 +0200 Subject: [PATCH 061/165] Test CSO v1 v2 and ZSO interoperability --- Compression.Tests/Cso/CsoTests.cs | 267 ++++++++++++++++++------------ 1 file changed, 160 insertions(+), 107 deletions(-) diff --git a/Compression.Tests/Cso/CsoTests.cs b/Compression.Tests/Cso/CsoTests.cs index 5ae960eec..5c0108fa4 100644 --- a/Compression.Tests/Cso/CsoTests.cs +++ b/Compression.Tests/Cso/CsoTests.cs @@ -1,126 +1,111 @@ using System.Buffers.Binary; +using Compression.Registry; +using FileFormat.Cso; namespace Compression.Tests.Cso; [TestFixture] public class CsoTests { - - // Build a minimal synthetic CSO (or ZSO): 24-byte header, N+1 index entries, then fake block data. - // uncompressed_size = 4096, block_size = 2048 => 2 blocks, 3 index entries. - private static byte[] BuildSyntheticImage(bool isZso, byte[] block0, byte[] block1, bool block0Stored = false, bool block1Stored = false) { - using var ms = new MemoryStream(); - ms.Write(isZso ? "ZISO"u8 : "CISO"u8); - Span buf = stackalloc byte[8]; - // header_size = 0x18 (24) - BinaryPrimitives.WriteUInt32LittleEndian(buf[..4], 0x18u); - ms.Write(buf[..4]); - // uncompressed_size = 4096 - BinaryPrimitives.WriteUInt64LittleEndian(buf, 4096UL); - ms.Write(buf); - // block_size = 2048 - BinaryPrimitives.WriteUInt32LittleEndian(buf[..4], 2048u); - ms.Write(buf[..4]); - // version, align, reserved[2] - ms.WriteByte(1); - ms.WriteByte(0); - ms.WriteByte(0); - ms.WriteByte(0); - - // Index table: 3 entries. Offsets point to byte positions in the file. - var indexStart = (uint)ms.Position; - var indexSize = 3u * 4u; // 12 bytes - var block0Offset = indexStart + indexSize; // immediately after index - var block1Offset = block0Offset + (uint)block0.Length; - var endOffset = block1Offset + (uint)block1.Length; - - uint entry0 = block0Offset | (block0Stored ? 0x8000_0000u : 0u); - uint entry1 = block1Offset | (block1Stored ? 0x8000_0000u : 0u); - uint entry2 = endOffset; - - BinaryPrimitives.WriteUInt32LittleEndian(buf[..4], entry0); - ms.Write(buf[..4]); - BinaryPrimitives.WriteUInt32LittleEndian(buf[..4], entry1); - ms.Write(buf[..4]); - BinaryPrimitives.WriteUInt32LittleEndian(buf[..4], entry2); - ms.Write(buf[..4]); - - ms.Write(block0); - ms.Write(block1); - return ms.ToArray(); - } + private const int BlockSize = 2048; [Test, Category("HappyPath")] public void Descriptor_Properties() { - var d = new FileFormat.Cso.CsoFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("Cso")); - Assert.That(d.Extensions, Contains.Item(".cso")); - Assert.That(d.Extensions, Contains.Item(".ziso")); - Assert.That(d.Category, Is.EqualTo(Compression.Registry.FormatCategory.Archive)); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(2)); + var descriptor = new CsoFormatDescriptor(); + Assert.That(descriptor.Id, Is.EqualTo("Cso")); + Assert.That(descriptor.Extensions, Does.Contain(".cso")); + Assert.That(descriptor.Extensions, Does.Contain(".ziso")); + Assert.That(descriptor.Extensions, Does.Contain(".zso")); + Assert.That(descriptor.Category, Is.EqualTo(FormatCategory.Archive)); + Assert.That(descriptor.MagicSignatures, Has.Count.EqualTo(2)); } - [Test, Category("HappyPath")] - public void List_SurfacesBlocksAndIndex() { - var block0 = new byte[] { 0x10, 0x20, 0x30 }; - var block1 = new byte[] { 0xAA, 0xBB }; - var img = BuildSyntheticImage(isZso: false, block0, block1, block0Stored: false, block1Stored: true); - - var desc = new FileFormat.Cso.CsoFormatDescriptor(); - using var ms = new MemoryStream(img); - var entries = desc.List(ms, null); - - // FULL.cso, metadata.ini, index.bin, blocks (dir), block_00000.bin, block_00001.bin = 6 entries - Assert.That(entries, Has.Count.EqualTo(6)); - Assert.That(entries[0].Name, Is.EqualTo("FULL.cso")); - Assert.That(entries[1].Name, Is.EqualTo("metadata.ini")); - Assert.That(entries[2].Name, Is.EqualTo("index.bin")); - Assert.That(entries[2].OriginalSize, Is.EqualTo(12)); - Assert.That(entries[3].Name, Is.EqualTo("blocks")); - Assert.That(entries[3].IsDirectory, Is.True); - Assert.That(entries[4].Name, Is.EqualTo("blocks/block_00000.bin")); - Assert.That(entries[4].OriginalSize, Is.EqualTo(block0.Length)); - Assert.That(entries[4].Method, Is.EqualTo("Deflate")); - Assert.That(entries[5].Name, Is.EqualTo("blocks/block_00001.bin")); - Assert.That(entries[5].OriginalSize, Is.EqualTo(block1.Length)); - Assert.That(entries[5].Method, Is.EqualTo("Stored")); + [TestCase(null, "CISO", 1, TestName = "CreateRoundTrip_CsoV1")] + [TestCase("zso", "ZISO", 1, TestName = "CreateRoundTrip_Zso")] + [TestCase("cso2", "CISO", 2, TestName = "CreateRoundTrip_CsoV2")] + [Category("RoundTrip")] + public void CreateAndLogicalBlockExtraction_RoundTrips(string? method, string magic, byte version) { + var payload = BuildPayload(BlockSize * 3 + 317); + var image = CreateImage(payload, method); + + Assert.That(System.Text.Encoding.ASCII.GetString(image, 0, 4), Is.EqualTo(magic)); + Assert.That(image[20], Is.EqualTo(version)); + Assert.That(ReadLogicalPayload(image), Is.EqualTo(payload)); + } + + [Test, Category("Interoperability")] + public void Zso_DecodesLibLz4KnownBlock() { + // liblz4 raw-block oracle for 2048 zero bytes (no frame header / no stored size prefix). + var encoded = Convert.FromHexString("1F000100FFFFFFFFFFFFFFEE500000000000"); + var image = BuildSingleBlockImage("ZISO", 1, encoded, methodFlag: false); + + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image); + var block = descriptor.ExtractEntryToMemory(stream, "blocks/block_00000.bin", null); + Assert.That(block, Is.EqualTo(new byte[BlockSize])); + + stream.Position = 0; + Assert.That(descriptor.List(stream, null).Single(e => e.Name == "blocks/block_00000.bin").Method, + Is.EqualTo("LZ4")); + } + + [Test, Category("Interoperability")] + public void CsoV2_DecodesLibLz4KnownBlock() { + var encoded = Convert.FromHexString("1F000100FFFFFFFFFFFFFFEE500000000000"); + var image = BuildSingleBlockImage("CISO", 2, encoded, methodFlag: true); + + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image); + var block = descriptor.ExtractEntryToMemory(stream, "blocks/block_00000.bin", null); + Assert.That(block, Is.EqualTo(new byte[BlockSize])); + + stream.Position = 0; + Assert.That(descriptor.List(stream, null).Single(e => e.Name == "blocks/block_00000.bin").Method, + Is.EqualTo("LZ4")); + } + + [Test, Category("Compatibility")] + public void CsoVersionZero_IsAcceptedAsV1() { + var payload = BuildPayload(BlockSize); + var image = CreateImage(payload); + image[20] = 0; + + Assert.That(ReadLogicalPayload(image), Is.EqualTo(payload)); } [Test, Category("HappyPath")] - public void List_ZsoReportsLz4Method() { - var img = BuildSyntheticImage(isZso: true, new byte[] { 1, 2 }, new byte[] { 3, 4 }); - var desc = new FileFormat.Cso.CsoFormatDescriptor(); - using var ms = new MemoryStream(img); - var entries = desc.List(ms, null); - Assert.That(entries[0].Name, Is.EqualTo("FULL.ziso")); - Assert.That(entries[4].Method, Is.EqualTo("LZ4")); + public void List_ReportsLogicalAndPhysicalBlockSizes() { + var payload = new byte[BlockSize + 137]; + var image = CreateImage(payload, "zso"); + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image); + + var entries = descriptor.List(stream, null); + var first = entries.Single(e => e.Name == "blocks/block_00000.bin"); + var last = entries.Single(e => e.Name == "blocks/block_00001.bin"); + Assert.That(first.OriginalSize, Is.EqualTo(BlockSize)); + Assert.That(last.OriginalSize, Is.EqualTo(137)); + Assert.That(first.CompressedSize, Is.LessThan(BlockSize)); + Assert.That(first.Method, Is.EqualTo("LZ4")); } [Test, Category("HappyPath")] - public void Extract_WritesBlocksAndMetadata() { - var block0 = new byte[] { 0xDE, 0xAD, 0xBE, 0xEF }; - var block1 = new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 }; - var img = BuildSyntheticImage(isZso: false, block0, block1); - - var desc = new FileFormat.Cso.CsoFormatDescriptor(); - var tmp = Path.Combine(Path.GetTempPath(), "cso_test_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(tmp); + public void Extract_WritesMetadataAndLogicalBlocks() { + var payload = BuildPayload(BlockSize * 2); + var image = CreateImage(payload, "cso2"); + var descriptor = new CsoFormatDescriptor(); + var directory = Path.Combine(Path.GetTempPath(), "cso_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); try { - using var ms = new MemoryStream(img); - desc.Extract(ms, tmp, null, null); - - Assert.That(File.Exists(Path.Combine(tmp, "FULL.cso")), Is.True); - Assert.That(File.ReadAllBytes(Path.Combine(tmp, "FULL.cso")), Is.EqualTo(img)); - Assert.That(File.Exists(Path.Combine(tmp, "metadata.ini")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "index.bin")), Is.True); - Assert.That(File.ReadAllBytes(Path.Combine(tmp, "blocks/block_00000.bin")), Is.EqualTo(block0)); - Assert.That(File.ReadAllBytes(Path.Combine(tmp, "blocks/block_00001.bin")), Is.EqualTo(block1)); - var meta = File.ReadAllText(Path.Combine(tmp, "metadata.ini")); - Assert.That(meta, Does.Contain("uncompressed_size=4096")); - Assert.That(meta, Does.Contain("block_size=2048")); - Assert.That(meta, Does.Contain("block_count=2")); - Assert.That(meta, Does.Contain("is_zso=0")); + using var stream = new MemoryStream(image); + descriptor.Extract(stream, directory, null, ["metadata.ini", "block_00001.bin"]); + + var metadata = File.ReadAllText(Path.Combine(directory, "metadata.ini")); + Assert.That(metadata, Does.Contain("variant=cso2")); + Assert.That(metadata, Does.Contain("block_size=2048")); + Assert.That(File.ReadAllBytes(Path.Combine(directory, "blocks/block_00001.bin")), + Is.EqualTo(payload.AsSpan(BlockSize, BlockSize).ToArray())); } finally { - Directory.Delete(tmp, recursive: true); + Directory.Delete(directory, recursive: true); } } @@ -128,8 +113,76 @@ public void Extract_WritesBlocksAndMetadata() { public void List_RejectsInvalidMagic() { var bogus = new byte[64]; bogus[0] = (byte)'X'; - var desc = new FileFormat.Cso.CsoFormatDescriptor(); - using var ms = new MemoryStream(bogus); - Assert.That(() => desc.List(ms, null), Throws.InstanceOf()); + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(bogus); + Assert.That(() => descriptor.List(stream, null), Throws.InstanceOf()); + } + + [Test, Category("ErrorHandling")] + public void List_RejectsNonMonotonicIndex() { + var image = CreateImage(new byte[BlockSize * 2]); + var first = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(24, 4)); + BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(28, 4), first - 1); + + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image); + Assert.That(() => descriptor.List(stream, null), Throws.InstanceOf()); + } + + private static byte[] CreateImage(byte[] payload, string? method = null, int blockSize = BlockSize) { + var creator = (IArchiveCreatable)new CsoFormatDescriptor(); + using var stream = new MemoryStream(); + var options = new FormatCreateOptions(method); + options.FormatSpecific["BlockSize"] = blockSize.ToString(System.Globalization.CultureInfo.InvariantCulture); + creator.Create(stream, [ArchiveInputInfo.InMemory("disc.iso", payload)], options); + return stream.ToArray(); + } + + private static byte[] ReadLogicalPayload(byte[] image) { + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image, writable: false); + var names = descriptor.List(stream, null) + .Where(e => !e.IsDirectory && e.Name.StartsWith("blocks/block_", StringComparison.Ordinal)) + .OrderBy(e => e.Index) + .Select(e => e.Name) + .ToArray(); + using var output = new MemoryStream(); + foreach (var name in names) { + stream.Position = 0; + output.Write(descriptor.ExtractEntryToMemory(stream, name, null)); + } + return output.ToArray(); + } + + private static byte[] BuildSingleBlockImage(string magic, byte version, byte[] encoded, bool methodFlag) { + using var stream = new MemoryStream(); + stream.Write(System.Text.Encoding.ASCII.GetBytes(magic)); + Span word = stackalloc byte[8]; + BinaryPrimitives.WriteUInt32LittleEndian(word[..4], 24); + stream.Write(word[..4]); + BinaryPrimitives.WriteUInt64LittleEndian(word, BlockSize); + stream.Write(word); + BinaryPrimitives.WriteUInt32LittleEndian(word[..4], BlockSize); + stream.Write(word[..4]); + stream.WriteByte(version); + stream.WriteByte(0); + stream.WriteByte(0); + stream.WriteByte(0); + + const uint dataOffset = 32; + var first = dataOffset | (methodFlag ? 0x8000_0000u : 0u); + BinaryPrimitives.WriteUInt32LittleEndian(word[..4], first); + stream.Write(word[..4]); + BinaryPrimitives.WriteUInt32LittleEndian(word[..4], dataOffset + checked((uint)encoded.Length)); + stream.Write(word[..4]); + stream.Write(encoded); + return stream.ToArray(); + } + + private static byte[] BuildPayload(int length) { + var payload = new byte[length]; + for (var i = 0; i < payload.Length; ++i) + payload[i] = (byte)((i * 17 + i / 31) & 0xFF); + return payload; } } From 56d9e07fbcf30e4c7a94f7f97ee40fa3201dcba1 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:05:05 +0200 Subject: [PATCH 062/165] Read GlusterFS brick backing images via XFS and ext --- .../FileSystem.GlusterFs/GlusterFsReader.cs | 237 ++++++++++++------ 1 file changed, 158 insertions(+), 79 deletions(-) diff --git a/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs b/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs index bb3342d34..44dc4d78a 100644 --- a/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs +++ b/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs @@ -2,126 +2,205 @@ using System.Buffers.Binary; using System.Globalization; using System.Text; +using FileSystem.Ext; +using FileSystem.Xfs; namespace FileSystem.GlusterFs; /// -/// Stage 0 detection-only reader for GlusterFS — permanent honest -/// fallback. GlusterFS itself has no on-disk image format: a -/// brick is a normal directory on a local POSIX filesystem -/// (XFS / ext4 / ...) and volume files are stored at their normal -/// POSIX paths inside that directory. All GlusterFS-specific state -/// lives in extended attributes (the trusted.gfid, -/// trusted.glusterfs.dht, trusted.glusterfs.volume-id, -/// trusted.glusterfs.pathinfo namespace). +/// Reads a single GlusterFS brick backing-store image by delegating the native +/// on-disk filesystem to the existing XFS or ext2/3/4 reader. /// -/// Consequences: -/// -/// There is no superblock or brick header to parse. -/// Distribution / replication state (DHT hashing → brick -/// mapping, AFR replicate metadata, EC dispersed metadata, rebalance -/// bookkeeping) only exists across multiple bricks on multiple -/// hosts, not inside any single image. -/// An R/O promotion is fundamentally incompatible with this -/// project's image-stream contract — recognising a GlusterFS -/// "volume" would require walking a live POSIX directory tree and -/// reading xattrs through the host OS, which is outside the -/// Stream-based IArchiveFormatOperations surface. -/// +/// GlusterFS does not define a separate block format. A brick is an export +/// directory on an ordinary filesystem that supports extended attributes. The +/// logical volume namespace and DHT/AFR/EC state span multiple bricks, while +/// object identity and translator metadata are stored in xattrs such as +/// trusted.gfid and trusted.glusterfs.*. Consequently this reader +/// intentionally exposes only the physical view of one supplied backing image. +/// It does not claim to reconstruct a Gluster volume from one brick. /// -/// The 0xCAFE5BAB magic verified by is a -/// workbench-internal probe convention used to dump and round-trip -/// hand-crafted "brick object" experiments; it is not a real -/// on-disk GlusterFS marker and no real GlusterFS deployment produces -/// it. The reader therefore stays a thin two-entry detector -/// (synthetic metadata.ini + raw gluster-brick.bin) and -/// will never grow real semantics. +/// The backing readers currently do not interpret or mutate Gluster xattrs. +/// Those bytes remain untouched because this type is read-only. The internal +/// .glusterfs GFID index is omitted from the normal brick listing so it is +/// not mistaken for user namespace content. /// public sealed class GlusterFsReader : IDisposable { - /// - /// Workbench-internal probe magic (0xCA 0xFE 0x5B 0xAB, 0xCAFE5BAB - /// big-endian) used by the detector tests. Not a real - /// GlusterFS structure — GlusterFS has no on-disk header at all. - /// - public static readonly byte[] BrickMagic = [0xCA, 0xFE, 0x5B, 0xAB]; + private const int ExtSuperblockOffset = 1024; + private const int ExtMagicOffset = ExtSuperblockOffset + 56; + private const ushort ExtMagic = 0xEF53; - private const int HeaderSize = 8; - - private readonly byte[] _data; + private readonly Stream _image; + private readonly bool _ownsImage; private readonly List _entries = []; + private XfsReader? _xfsReader; + private ExtReader? _extReader; /// - /// Gets the entries. + /// Gets the entries in the single-brick physical view. /// public IReadOnlyList Entries => _entries; + /// - /// Gets or sets the magic word. - /// - public uint MagicWord { get; private set; } - /// - /// Gets or sets the trailing word. + /// Gets a value indicating whether a supported backing filesystem was found. /// - public uint TrailingWord { get; private set; } + public bool ValidHeader { get; private set; } + /// - /// Gets a value indicating whether valid header. + /// Gets the detected backing filesystem name (xfs or ext). /// - public bool ValidHeader { get; private set; } + public string BackingFileSystem { get; private set; } = ""; /// - /// Initializes a new instance of . + /// Initializes a reader over one brick backing-store image. /// public GlusterFsReader(Stream stream) { - using var ms = new MemoryStream(); - stream.CopyTo(ms); - _data = ms.ToArray(); + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead) + throw new ArgumentException("GlusterFS brick image must be readable.", nameof(stream)); + + if (stream.CanSeek) { + _image = stream; + } else { + var buffered = new MemoryStream(); + stream.CopyTo(buffered); + buffered.Position = 0; + _image = buffered; + _ownsImage = true; + } + Parse(); } private void Parse() { - if (_data.Length < HeaderSize) - throw new InvalidDataException("GlusterFS: file too small for brick object header."); + if (IsXfs()) { + ParseXfs(); + return; + } + + if (IsExt()) { + ParseExt(); + return; + } + + throw new InvalidDataException( + "GlusterFS: input is not a supported brick backing-store image (expected XFS or ext2/3/4). " + + "GlusterFS itself has no standalone image magic; use a backing-filesystem image of one brick."); + } + + private bool IsXfs() { + if (_image.Length < 4) return false; + Span magic = stackalloc byte[4]; + ReadAt(0, magic); + return magic.SequenceEqual("XFSB"u8); + } + + private bool IsExt() { + if (_image.Length < ExtMagicOffset + sizeof(ushort)) return false; + Span magic = stackalloc byte[sizeof(ushort)]; + ReadAt(ExtMagicOffset, magic); + return BinaryPrimitives.ReadUInt16LittleEndian(magic) == ExtMagic; + } + + private void ParseXfs() { + _image.Position = 0; + _xfsReader = new XfsReader(_image, leaveOpen: true); + this.BackingFileSystem = "xfs"; + this.ValidHeader = true; + AddMetadata(_xfsReader.Entries.Count); - if (!_data.AsSpan(0, 4).SequenceEqual(BrickMagic)) - throw new InvalidDataException("GlusterFS: missing 0xCAFE5BAB brick magic at offset 0."); + foreach (var entry in _xfsReader.Entries) { + if (IsGlusterInternal(entry.Name)) continue; + var captured = entry; + _entries.Add(new GlusterFsEntry { + Name = BrickPath(entry.Name), + Size = entry.Size, + IsDirectory = entry.IsDirectory, + DataFactory = entry.IsDirectory ? null : () => _xfsReader.Extract(captured), + }); + } + } - this.MagicWord = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(0, 4)); - this.TrailingWord = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(4, 4)); + private void ParseExt() { + _image.Position = 0; + _extReader = new ExtReader(_image, leaveOpen: true); + this.BackingFileSystem = "ext"; this.ValidHeader = true; + AddMetadata(_extReader.Entries.Count); + + foreach (var entry in _extReader.Entries) { + if (IsGlusterInternal(entry.Name)) continue; + var captured = entry; + _entries.Add(new GlusterFsEntry { + Name = BrickPath(entry.Name), + Size = entry.Size, + IsDirectory = entry.IsDirectory, + DataFactory = entry.IsDirectory ? null : () => _extReader.Extract(captured), + }); + } + } - var meta = BuildMetadata(); - _entries.Add(new GlusterFsEntry { Name = "metadata.ini", Size = meta.Length, IsDirectory = false, Offset = 0, Data = meta }); - _entries.Add(new GlusterFsEntry { Name = "gluster-brick.bin", Size = _data.Length, IsDirectory = false, Offset = 0, Data = _data }); + private void AddMetadata(int backingEntryCount) { + var metadata = BuildMetadata(backingEntryCount); + _entries.Add(new GlusterFsEntry { + Name = "metadata.ini", + Size = metadata.Length, + Data = metadata, + }); } - private byte[] BuildMetadata() { - var bldr = new StringBuilder(); - bldr.Append("parse_status=detection-only\n"); - bldr.Append("format=GlusterFS (no on-disk image; brick = normal directory on local FS)\n"); - bldr.Append(CultureInfo.InvariantCulture, $"magic_word=0x{this.MagicWord:X8}\n"); - bldr.Append("magic_offset=0\n"); - bldr.Append("magic_kind=workbench-internal probe (NOT a real GlusterFS marker)\n"); - bldr.Append(CultureInfo.InvariantCulture, $"trailing_word=0x{this.TrailingWord:X8}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"image_size={_data.Length}\n"); - bldr.Append("stage=0\n"); - bldr.Append("stage_permanent=true\n"); - bldr.Append("ro_blocked_reason=GlusterFS has no on-disk image format; bricks are normal "); - bldr.Append("directories on XFS/ext4 and state lives in xattrs (trusted.gfid, "); - bldr.Append("trusted.glusterfs.*). Walking a live POSIX tree + reading xattrs is outside "); - bldr.Append("the image-stream contract.\n"); - return Encoding.UTF8.GetBytes(bldr.ToString()); + private byte[] BuildMetadata(int backingEntryCount) { + var builder = new StringBuilder(); + builder.Append("parse_status=single-brick-read-only\n"); + builder.Append("format=GlusterFS brick backing store\n"); + builder.Append(CultureInfo.InvariantCulture, $"backing_fs={this.BackingFileSystem}\n"); + builder.Append(CultureInfo.InvariantCulture, $"image_size={_image.Length}\n"); + builder.Append(CultureInfo.InvariantCulture, $"backing_entry_count={backingEntryCount}\n"); + builder.Append("view=physical single-brick namespace\n"); + builder.Append("gluster_internal_directory=.glusterfs (hidden from normal listing)\n"); + builder.Append("xattrs=preserved in image but not interpreted\n"); + builder.Append("cluster_namespace_reconstruction=false\n"); + builder.Append("mutation=false\n"); + builder.Append("mutation_blocker=Gluster object identity and DHT/AFR/EC state live in trusted.gfid/trusted.glusterfs.* xattrs; current backing writers do not preserve those attributes during rebuilds.\n"); + return Encoding.UTF8.GetBytes(builder.ToString()); + } + + private static string BrickPath(string path) { + var normalized = path.Replace('\\', '/').TrimStart('/'); + return "brick/" + normalized; + } + + private static bool IsGlusterInternal(string path) { + var normalized = path.Replace('\\', '/').Trim('/'); + return normalized.Equals(".glusterfs", StringComparison.Ordinal) || + normalized.StartsWith(".glusterfs/", StringComparison.Ordinal); + } + + private void ReadAt(long offset, Span destination) { + var original = _image.Position; + try { + _image.Position = offset; + _image.ReadExactly(destination); + } finally { + _image.Position = original; + } } /// - /// Decodes the supplied input. + /// Extracts one surfaced entry. /// public byte[] Extract(GlusterFsEntry entry) { ArgumentNullException.ThrowIfNull(entry); - return entry.Data; + if (entry.IsDirectory) return []; + return entry.DataFactory?.Invoke() ?? entry.Data; } /// - /// Releases resources held by this instance. + /// Releases backing filesystem readers and any spool created for a non-seekable input. /// - public void Dispose() { } + public void Dispose() { + _xfsReader?.Dispose(); + _extReader?.Dispose(); + if (_ownsImage) _image.Dispose(); + } } From 5dfb863c0d0d9e99867d0c521d9495d591b576b3 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:05:13 +0200 Subject: [PATCH 063/165] + make Apple IPSW lossless and maintainable Preserve exact ZIP member paths instead of flattening firmware paths, stream file-backed IPSW creation, and add transactional rebuild-based defrag/shrink while retaining existing wipe/purge support. Harden ZIP layout mapping for general-purpose-bit-3 data descriptors so wipe never mistakes valid descriptor metadata for dead space. Add IPSW regression coverage for path round-trips, defrag, shrink, purge, descriptor-safe wipe, and streamed creation. --- .../Ipsw/IpswMaintenanceTests.cs | 180 +++++ Compression.Tests/Ipsw/IpswTests.cs | 105 +-- .../FileFormat.Ipsw/IpswFormatDescriptor.cs | 637 +++++++++++------- FileFormats/FileFormat.Zip/ZipLayoutMap.cs | 266 ++++++-- 4 files changed, 821 insertions(+), 367 deletions(-) create mode 100644 Compression.Tests/Ipsw/IpswMaintenanceTests.cs diff --git a/Compression.Tests/Ipsw/IpswMaintenanceTests.cs b/Compression.Tests/Ipsw/IpswMaintenanceTests.cs new file mode 100644 index 000000000..b4c72815e --- /dev/null +++ b/Compression.Tests/Ipsw/IpswMaintenanceTests.cs @@ -0,0 +1,180 @@ +using System.IO.Compression; +using Compression.Registry; +using FileFormat.Ipsw; + +namespace Compression.Tests.Ipsw; + +[TestFixture] +public class IpswMaintenanceTests { + + [Test, Category("HappyPath")] + public void Descriptor_AdvertisesFeasibleMaintenanceContracts() { + var descriptor = new IpswFormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + }); + } + + [Test, Category("RoundTrip")] + public void Defragment_RemovesDeadZipSpaceAndPreservesLiveMembers() { + var descriptor = new IpswFormatDescriptor(); + using var image = BuildIpsw( + ("BuildManifest.plist", ""u8.ToArray()), + ("Firmware/dead.bin", Enumerable.Repeat((byte)0xA5, 64 * 1024).ToArray()), + ("Firmware/dfu/iBSS.test.RELEASE.im4p", [0xDE, 0xAD, 0xBE, 0xEF])); + + ((IArchiveModifiable)descriptor).Remove(image, ["Firmware/dead.bin"]); + var beforeLength = image.Length; + + ((IArchiveDefragmentable)descriptor).Defragment(image); + + Assert.That(image.Length, Is.LessThan(beforeLength)); + Assert.That(ReadEntry(image, "Firmware/dfu/iBSS.test.RELEASE.im4p"), Is.EqualTo(new byte[] { 0xDE, 0xAD, 0xBE, 0xEF })); + Assert.That(RawNames(image), Does.Not.Contain("Firmware/dead.bin")); + Assert.That(RawNames(image), Does.Contain("Firmware/dfu/iBSS.test.RELEASE.im4p")); + } + + [Test, Category("RoundTrip")] + public void Shrink_TightPacksDeadZipSpaceAndPreservesLiveMembers() { + var descriptor = new IpswFormatDescriptor(); + using var image = BuildIpsw( + ("BuildManifest.plist", ""u8.ToArray()), + ("Firmware/dead.bin", Enumerable.Repeat((byte)0x5A, 64 * 1024).ToArray()), + ("Restore.plist", "restore"u8.ToArray())); + ((IArchiveModifiable)descriptor).Remove(image, ["Firmware/dead.bin"]); + + using var output = new MemoryStream(); + ((IArchiveShrinkable)descriptor).Shrink(image, output); + + Assert.That(output.Length, Is.LessThan(image.Length)); + Assert.That(ReadEntry(output, "Restore.plist"), Is.EqualTo("restore"u8.ToArray())); + Assert.That(RawNames(output), Is.EquivalentTo(new[] { "BuildManifest.plist", "Restore.plist" })); + } + + [Test, Category("RoundTrip")] + public void Purge_LeavesAValidEmptyIpswContainer() { + var descriptor = new IpswFormatDescriptor(); + using var image = BuildIpsw( + ("BuildManifest.plist", ""u8.ToArray()), + ("Firmware/dfu/iBSS.test.RELEASE.im4p", [1, 2, 3, 4])); + + ((IArchivePurgeable)descriptor).Purge(image); + + Assert.That(RawNames(image), Is.Empty); + image.Position = 0; + var listed = descriptor.List(image, null).Select(entry => entry.Name).ToArray(); + Assert.That(listed, Is.EquivalentTo(new[] { "FULL.ipsw", "metadata.ini" })); + } + + [Test, Category("RoundTrip")] + public void Wipe_PreservesZipDataDescriptors() { + var original = BuildDataDescriptorIpsw(); + using var image = new MemoryStream(original.ToArray()); + var descriptor = new IpswFormatDescriptor(); + + var layout = ((IArchiveLayoutMap)descriptor).EnumerateLayout(image).ToArray(); + Assert.That(layout, Has.Some.Matches(block => + block.Kind == DefragBlockKind.MetadataReserved && + block.FileName?.StartsWith("Data descriptor:", StringComparison.Ordinal) == true)); + + image.Position = 0; + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(image); + + Assert.Multiple(() => { + Assert.That(wiped, Is.Zero); + Assert.That(image.ToArray(), Is.EqualTo(original)); + }); + } + + [Test, Category("RoundTrip")] + public void Create_FileBackedInputStreamsPayloadAndPreservesNestedPath() { + var temp = Path.GetTempFileName(); + try { + var payload = Enumerable.Range(0, 1024 * 1024).Select(i => (byte)(i * 31)).ToArray(); + File.WriteAllBytes(temp, payload); + var descriptor = new IpswFormatDescriptor(); + using var image = new MemoryStream(); + + ((IArchiveCreatable)descriptor).Create(image, [ + ArchiveInputInfo.FromFile(new FileInfo(temp), "Firmware/dfu/streamed.bin"), + ], new FormatCreateOptions()); + + Assert.That(ReadEntry(image, "Firmware/dfu/streamed.bin"), Is.EqualTo(payload)); + image.Position = 0; + Assert.That(descriptor.List(image, null).Select(entry => entry.Name), Does.Contain("Firmware/dfu/streamed.bin")); + } finally { + File.Delete(temp); + } + } + + private static MemoryStream BuildIpsw(params (string Name, byte[] Data)[] entries) { + var image = new MemoryStream(); + using (var zip = new ZipArchive(image, ZipArchiveMode.Create, leaveOpen: true)) { + foreach (var (name, data) in entries) { + var entry = zip.CreateEntry(name, CompressionLevel.NoCompression); + using var stream = entry.Open(); + stream.Write(data); + } + } + image.Position = 0; + return image; + } + + private static byte[] BuildDataDescriptorIpsw() { + using var backing = new MemoryStream(); + using (var forwardOnly = new ForwardOnlyWriteStream(backing)) + using (var zip = new ZipArchive(forwardOnly, ZipArchiveMode.Create, leaveOpen: true)) { + var entry = zip.CreateEntry("BuildManifest.plist", CompressionLevel.Optimal); + using var stream = entry.Open(); + stream.Write(Enumerable.Repeat((byte)'A', 4096).ToArray()); + } + return backing.ToArray(); + } + + private static string[] RawNames(Stream image) { + image.Position = 0; + using var zip = new ZipArchive(image, ZipArchiveMode.Read, leaveOpen: true); + return zip.Entries.Select(entry => entry.FullName).ToArray(); + } + + private static byte[] ReadEntry(Stream image, string name) { + image.Position = 0; + using var zip = new ZipArchive(image, ZipArchiveMode.Read, leaveOpen: true); + var entry = zip.GetEntry(name) ?? throw new KeyNotFoundException(name); + using var source = entry.Open(); + using var output = new MemoryStream(); + source.CopyTo(output); + return output.ToArray(); + } + + private sealed class ForwardOnlyWriteStream(Stream inner) : Stream { + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => throw new NotSupportedException(); } + public override void Flush() => inner.Flush(); + public override Task FlushAsync(CancellationToken cancellationToken) => inner.FlushAsync(cancellationToken); + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => inner.Write(buffer, offset, count); + public override void Write(ReadOnlySpan buffer) => inner.Write(buffer); + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + => inner.WriteAsync(buffer, cancellationToken); + + protected override void Dispose(bool disposing) { + if (disposing) + inner.Flush(); + base.Dispose(disposing); + } + } +} diff --git a/Compression.Tests/Ipsw/IpswTests.cs b/Compression.Tests/Ipsw/IpswTests.cs index da535ab9e..b6c795774 100644 --- a/Compression.Tests/Ipsw/IpswTests.cs +++ b/Compression.Tests/Ipsw/IpswTests.cs @@ -15,10 +15,10 @@ private static byte[] BuildSyntheticIpsw() { "ProductBuildVersion21E236" + "ProductTypeiPhone15,3" + "")); - WriteEntry(zip, "Firmware/sep-firmware.bin", new byte[] { 0xA, 0xB, 0xC }); - WriteEntry(zip, "Firmware/dfu/iBSS.d22.RELEASE.im4p", new byte[] { 0xD }); - WriteEntry(zip, "LLB.iphone15,3.RELEASE.im4p", new byte[] { 0xE }); - WriteEntry(zip, "iBoot.iphone15,3.RELEASE.im4p", new byte[] { 0xF }); + WriteEntry(zip, "Firmware/sep-firmware.bin", [0xA, 0xB, 0xC]); + WriteEntry(zip, "Firmware/dfu/iBSS.d22.RELEASE.im4p", [0xD]); + WriteEntry(zip, "LLB.iphone15,3.RELEASE.im4p", [0xE]); + WriteEntry(zip, "iBoot.iphone15,3.RELEASE.im4p", [0xF]); WriteEntry(zip, "058-90000-000.dmg", Enumerable.Repeat((byte)0x55, 32).ToArray()); WriteEntry(zip, "Restore.plist", "some plist"u8.ToArray()); } @@ -27,64 +27,71 @@ private static byte[] BuildSyntheticIpsw() { private static void WriteEntry(ZipArchive zip, string name, byte[] data) { var entry = zip.CreateEntry(name, CompressionLevel.NoCompression); - using var es = entry.Open(); - es.Write(data); + using var stream = entry.Open(); + stream.Write(data); } [Test, Category("HappyPath")] public void Descriptor_Properties() { - var d = new FileFormat.Ipsw.IpswFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("Ipsw")); - Assert.That(d.CompoundExtensions, Contains.Item(".ipsw")); - Assert.That(d.Extensions, Is.Empty); - Assert.That(d.MagicSignatures, Is.Empty); - Assert.That(d.Category, Is.EqualTo(Compression.Registry.FormatCategory.Archive)); + var descriptor = new FileFormat.Ipsw.IpswFormatDescriptor(); + Assert.That(descriptor.Id, Is.EqualTo("Ipsw")); + Assert.That(descriptor.CompoundExtensions, Contains.Item(".ipsw")); + Assert.That(descriptor.Extensions, Is.Empty); + Assert.That(descriptor.MagicSignatures, Is.Empty); + Assert.That(descriptor.Category, Is.EqualTo(Compression.Registry.FormatCategory.Archive)); } - [Test, Category("HappyPath")] - public void List_SurfacesCanonicalAppleArtifacts() { - var ipsw = BuildSyntheticIpsw(); - var desc = new FileFormat.Ipsw.IpswFormatDescriptor(); - using var ms = new MemoryStream(ipsw); - var entries = desc.List(ms, null); + [Test, Category("HappyPath"), Category("RoundTrip")] + public void List_PreservesExactZipPaths() { + var descriptor = new FileFormat.Ipsw.IpswFormatDescriptor(); + using var image = new MemoryStream(BuildSyntheticIpsw()); + + var names = descriptor.List(image, null).Select(entry => entry.Name).ToList(); - var names = entries.Select(e => e.Name).ToList(); - Assert.That(names, Contains.Item("FULL.ipsw")); - Assert.That(names, Contains.Item("metadata.ini")); - Assert.That(names, Contains.Item("BuildManifest.plist")); - Assert.That(names, Has.Some.EqualTo("Firmware/sep-firmware.bin")); - Assert.That(names, Has.Some.EqualTo("Firmware/iBSS.d22.RELEASE.im4p")); - Assert.That(names, Contains.Item("LLB.iphone15,3.RELEASE.im4p")); - Assert.That(names, Contains.Item("iBoot.iphone15,3.RELEASE.im4p")); - Assert.That(names, Contains.Item("058-90000-000.dmg")); - Assert.That(names, Has.Some.EqualTo("other/Restore.plist")); + Assert.Multiple(() => { + Assert.That(names, Contains.Item("FULL.ipsw")); + Assert.That(names, Contains.Item("metadata.ini")); + Assert.That(names, Contains.Item("BuildManifest.plist")); + Assert.That(names, Contains.Item("Firmware/sep-firmware.bin")); + Assert.That(names, Contains.Item("Firmware/dfu/iBSS.d22.RELEASE.im4p")); + Assert.That(names, Contains.Item("LLB.iphone15,3.RELEASE.im4p")); + Assert.That(names, Contains.Item("iBoot.iphone15,3.RELEASE.im4p")); + Assert.That(names, Contains.Item("058-90000-000.dmg")); + Assert.That(names, Contains.Item("Restore.plist")); + Assert.That(names, Does.Not.Contain("Firmware/iBSS.d22.RELEASE.im4p")); + Assert.That(names, Does.Not.Contain("other/Restore.plist")); + }); } - [Test, Category("HappyPath")] - public void Extract_WritesCanonicalArtifactsAndMetadata() { - var ipsw = BuildSyntheticIpsw(); - var desc = new FileFormat.Ipsw.IpswFormatDescriptor(); - var tmp = Path.Combine(Path.GetTempPath(), "ipsw_test_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(tmp); + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Extract_PreservesZipPathsAndRendersMetadata() { + var descriptor = new FileFormat.Ipsw.IpswFormatDescriptor(); + var temp = Path.Combine(Path.GetTempPath(), "ipsw_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(temp); try { - using var ms = new MemoryStream(ipsw); - desc.Extract(ms, tmp, null, null); + using var image = new MemoryStream(BuildSyntheticIpsw()); + descriptor.Extract(image, temp, null, null); - Assert.That(File.Exists(Path.Combine(tmp, "FULL.ipsw")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "BuildManifest.plist")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "Firmware", "sep-firmware.bin")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "LLB.iphone15,3.RELEASE.im4p")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "iBoot.iphone15,3.RELEASE.im4p")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "058-90000-000.dmg")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "other", "Restore.plist")), Is.True); + Assert.Multiple(() => { + Assert.That(File.Exists(Path.Combine(temp, "FULL.ipsw")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "BuildManifest.plist")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "Firmware", "sep-firmware.bin")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "Firmware", "dfu", "iBSS.d22.RELEASE.im4p")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "LLB.iphone15,3.RELEASE.im4p")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "iBoot.iphone15,3.RELEASE.im4p")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "058-90000-000.dmg")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "Restore.plist")), Is.True); + }); - var meta = File.ReadAllText(Path.Combine(tmp, "metadata.ini")); - Assert.That(meta, Does.Contain("product_version=17.4.1")); - Assert.That(meta, Does.Contain("build_version=21E236")); - Assert.That(meta, Does.Contain("identifier=iPhone15,3")); - Assert.That(meta, Does.Contain("total_zip_entries=7")); + var metadata = File.ReadAllText(Path.Combine(temp, "metadata.ini")); + Assert.Multiple(() => { + Assert.That(metadata, Does.Contain("product_version=17.4.1")); + Assert.That(metadata, Does.Contain("build_version=21E236")); + Assert.That(metadata, Does.Contain("identifier=iPhone15,3")); + Assert.That(metadata, Does.Contain("total_zip_entries=7")); + }); } finally { - Directory.Delete(tmp, recursive: true); + Directory.Delete(temp, recursive: true); } } } diff --git a/FileFormats/FileFormat.Ipsw/IpswFormatDescriptor.cs b/FileFormats/FileFormat.Ipsw/IpswFormatDescriptor.cs index 9b7fba649..72c956edc 100644 --- a/FileFormats/FileFormat.Ipsw/IpswFormatDescriptor.cs +++ b/FileFormats/FileFormat.Ipsw/IpswFormatDescriptor.cs @@ -3,316 +3,433 @@ using System.IO.Compression; using System.Text; using Compression.Registry; +using Compression.Registry.Streaming; using static Compression.Registry.FormatHelpers; namespace FileFormat.Ipsw; /// -/// Apple IPSW / OTA firmware package. An IPSW is just a ZIP file (with an Apple-specific layout). -/// Rather than surfacing entries as a flat generic ZIP, this descriptor lifts the well-known -/// Apple artifacts (BuildManifest.plist, Firmware/ subtree, LLB.*, iBSS.*, -/// iBEC.*, iBoot.*, root-filesystem *.dmg) into first-class canonical entries. -/// Everything else is exposed under other/. -/// -/// This is a compound-extension descriptor (.ipsw, .otazip): magic is empty so -/// it does not steal generic ZIPs. Read-only; the plist and DMG payloads are emitted as raw bytes -/// — no plist parsing or DMG mounting. -/// -/// References: -/// -/// https://theapplewiki.com — The Apple Wiki (formerly The iPhone Wiki) — community IPSW documentation -/// https://github.com/blacktop/ipsw — ipsw — maintained IPSW research and extraction tool -/// https://en.wikipedia.org/wiki/IPSW — Wikipedia -/// +/// Apple IPSW / OTA firmware package. IPSW files are ZIP containers whose entry paths are +/// semantically significant: manifests refer to firmware members by their ZIP path, so the +/// descriptor exposes those paths losslessly instead of projecting them into a flattened view. /// -public sealed class IpswFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveLayoutMap, IArchiveCreatable, IArchiveModifiable { +/// +/// The ZIP container itself is mutable. Inner DMG, IMG4/IM4P and other firmware payloads +/// remain opaque byte streams and are handled by their own descriptors when opened separately. +/// Rebuild maintenance (defrag/shrink) re-encodes only the ZIP envelope. Entry bytes and +/// entry paths are preserved; Apple restore tooling also accepts an extracted IPSW directory, +/// so ZIP physical offsets are not part of the firmware payload identity. +/// FULL.ipsw and metadata.ini are synthetic views reserved by this descriptor +/// and are never written back as ZIP members. +/// +public sealed class IpswFormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IArchiveLayoutMap, + IArchiveCreatable, + IArchiveModifiable, + IArchiveDefragmentable, + IArchiveShrinkable, + ISyntheticEntryNames { - /// - public IEnumerable EnumerateLayout(Stream archive) => FileFormat.Zip.ZipLayoutMap.Enumerate(archive); + private static readonly HashSet SyntheticEntries = new(StringComparer.OrdinalIgnoreCase) { + "FULL.ipsw", + "metadata.ini", + }; + + private sealed record IpswEntry( + string Name, + long Size, + long CompressedSize, + string Method, + bool IsDirectory, + DateTime? LastModified, + string Kind); + + public IReadOnlySet SyntheticEntryNames => SyntheticEntries; + + public IEnumerable EnumerateLayout(Stream archive) + => FileFormat.Zip.ZipLayoutMap.Enumerate(archive); - /// - /// Gets the id. - /// public string Id => "Ipsw"; - /// - /// Gets the display name. - /// public string DisplayName => "Apple IPSW"; - /// - /// Gets the category. - /// public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; - /// - /// Gets the default extension. - /// public string DefaultExtension => ".ipsw"; - /// - /// Gets the extensions. - /// public IReadOnlyList Extensions => []; - /// - /// Gets the compound extensions. - /// public IReadOnlyList CompoundExtensions => [".ipsw", ".otazip"]; - /// - /// Gets the magic signatures. - /// public IReadOnlyList MagicSignatures => []; - /// - /// Gets the methods. - /// public IReadOnlyList Methods => [new("deflate", "Deflate"), new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// public string Description => - "Apple firmware package (ZIP containing BuildManifest.plist, Firmware/, DMG root FS). " + - "R/W: in-place Add / Remove against the ZIP central directory (delegates to ZipModifier). " + - "Inner DMG / firmware blob mutation is delegated to FileFormat.Dmg and the per-firmware descriptors."; + "Apple firmware package (ZIP containing BuildManifest.plist, Firmware/, boot firmware and disk images). " + + "R/W edits operate on the ZIP namespace; defrag/shrink rebuild the ZIP envelope without changing payload bytes."; - private sealed record CanonicalEntry(string CanonicalName, string ZipEntryName, long Size, string Method, DateTime? LastModified, string? Kind); - - /// - /// Lists the entries in the supplied container. - /// public List List(Stream stream, string? password) { - var (canonical, total) = EnumerateCanonical(stream); - var entries = new List(2 + canonical.Count); - entries.Add(new ArchiveEntryInfo(0, "FULL.ipsw", stream.Length, stream.Length, "Stored", false, false, null)); - entries.Add(new ArchiveEntryInfo(1, "metadata.ini", 0, 0, "Stored", false, false, null, Kind: $"total_zip_entries={total}")); - for (var i = 0; i < canonical.Count; ++i) { - var c = canonical[i]; - entries.Add(new ArchiveEntryInfo( - Index: 2 + i, - Name: c.CanonicalName, - OriginalSize: c.Size, - CompressedSize: c.Size, - Method: c.Method, - IsDirectory: false, - IsEncrypted: false, - LastModified: c.LastModified, - Kind: c.Kind)); + ArgumentNullException.ThrowIfNull(stream); + var entries = EnumerateEntries(stream); + var result = new List(entries.Count + SyntheticEntries.Count) { + new(0, "FULL.ipsw", stream.Length, stream.Length, "Stored", false, false, null, Kind: "container"), + new(1, "metadata.ini", 0, 0, "Stored", false, false, null, Kind: $"total_zip_entries={entries.Count}"), + }; + + foreach (var entry in entries) { + if (IsSynthetic(entry.Name)) + continue; + result.Add(new ArchiveEntryInfo( + result.Count, + entry.Name, + entry.Size, + entry.CompressedSize, + entry.Method, + entry.IsDirectory, + false, + entry.LastModified, + Kind: entry.Kind)); } - return entries; + + return result; } - /// - /// Decodes the supplied input. - /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - // FULL.ipsw: stream directly from the input — never materialize into memory. + ArgumentNullException.ThrowIfNull(stream); + ArgumentException.ThrowIfNullOrWhiteSpace(outputDir); + if (Wants(files, "FULL.ipsw")) { - stream.Seek(0, SeekOrigin.Begin); - var fullPath = Path.Combine(outputDir, "FULL.ipsw"); - var dir = Path.GetDirectoryName(fullPath); - if (dir != null) Directory.CreateDirectory(dir); - using var outStream = File.Create(fullPath); - stream.CopyTo(outStream); + stream.Position = 0; + var fullPath = SafeCombine(outputDir, "FULL.ipsw"); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + using var output = File.Create(fullPath); + stream.CopyTo(output); } - // Re-open ZIP on the seekable input stream. - stream.Seek(0, SeekOrigin.Begin); + stream.Position = 0; using var zip = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); - var (canonical, total) = EnumerateCanonicalFromZip(zip); - - string? identifier = null; - string? productVersion = null; - string? buildVersion = null; + foreach (var entry in zip.Entries) { + var name = NormalizeZipPath(entry.FullName); + if (IsSynthetic(name) || !Wants(files, name)) + continue; - foreach (var c in canonical) { - if (!Wants(files, c.CanonicalName)) continue; - var zipEntry = zip.GetEntry(c.ZipEntryName); - if (zipEntry == null) continue; - - var destPath = SafeCombine(outputDir, c.CanonicalName); - var destDir = Path.GetDirectoryName(destPath); - if (destDir != null) Directory.CreateDirectory(destDir); - using (var es = zipEntry.Open()) - using (var outFile = File.Create(destPath)) { - es.CopyTo(outFile); + var destination = SafeCombine(outputDir, name); + if (IsDirectoryEntry(entry)) { + Directory.CreateDirectory(destination); + continue; } - if (c.CanonicalName == "BuildManifest.plist") { - // BuildManifest is bounded (typically <1 MB) — safe to read back for metadata parsing. - var data = File.ReadAllBytes(destPath); - TryParsePlistFields(data, out identifier, out productVersion, out buildVersion); - } + var parent = Path.GetDirectoryName(destination); + if (!string.IsNullOrEmpty(parent)) + Directory.CreateDirectory(parent); + using var source = entry.Open(); + using var output = File.Create(destination); + source.CopyTo(output); } if (Wants(files, "metadata.ini")) { - // If we didn't extract the manifest above, best-effort parse it now to populate metadata. - if (identifier == null && productVersion == null && buildVersion == null) { - var manifest = zip.GetEntry("BuildManifest.plist"); - if (manifest != null) { - // Manifest is small and bounded — fine to materialize. - using var es = manifest.Open(); - using var ms = new MemoryStream(); - es.CopyTo(ms); - TryParsePlistFields(ms.ToArray(), out identifier, out productVersion, out buildVersion); - } - } - WriteFile(outputDir, "metadata.ini", - Encoding.UTF8.GetBytes(BuildMetadataIni(identifier, productVersion, buildVersion, total))); + var metadata = BuildMetadata(zip); + var metadataPath = SafeCombine(outputDir, "metadata.ini"); + Directory.CreateDirectory(Path.GetDirectoryName(metadataPath)!); + File.WriteAllBytes(metadataPath, metadata); } } - // ── IArchiveCreatable ───────────────────────────────────────────── + public Stream OpenEntry(Stream archive, string entryName, string? password) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentException.ThrowIfNullOrWhiteSpace(entryName); + + archive.Position = 0; + if (string.Equals(entryName, "FULL.ipsw", StringComparison.OrdinalIgnoreCase)) + return new BoundedEntryStream(archive, archive.Length, leaveOpen: true); + + if (string.Equals(entryName, "metadata.ini", StringComparison.OrdinalIgnoreCase)) { + using var zip = new ZipArchive(archive, ZipArchiveMode.Read, leaveOpen: true); + var metadata = BuildMetadata(zip); + return new BoundedEntryStream(new MemoryStream(metadata, writable: false), metadata.Length, leaveOpen: false); + } + + var reader = new ZipArchive(archive, ZipArchiveMode.Read, leaveOpen: true); + var normalized = NormalizeZipPath(entryName); + var entry = reader.Entries.FirstOrDefault(candidate => + string.Equals(NormalizeZipPath(candidate.FullName), normalized, StringComparison.OrdinalIgnoreCase)); + if (entry == null || IsDirectoryEntry(entry)) { + reader.Dispose(); + return new BoundedEntryStream(new MemoryStream([], writable: false), 0, leaveOpen: false); + } + + var owned = new ZipOwnedReadStream(entry.Open(), reader); + return new BoundedEntryStream(owned, entry.Length, leaveOpen: false); + } - /// - /// Emits a fresh IPSW (ZIP) container from the supplied inputs. Synthetic - /// canonical entries the descriptor surfaces on read (FULL.ipsw, - /// metadata.ini) are silently dropped — they aren't real ZIP entries. - /// All other inputs are stored under their . - /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { ArgumentNullException.ThrowIfNull(output); ArgumentNullException.ThrowIfNull(inputs); - using var zip = new FileFormat.Zip.ZipWriter(output, leaveOpen: true); - foreach (var (name, data) in FilesOnly(inputs)) { - if (string.Equals(name, "FULL.ipsw", StringComparison.OrdinalIgnoreCase)) continue; - if (string.Equals(name, "metadata.ini", StringComparison.OrdinalIgnoreCase)) continue; - zip.AddEntry(name, data); + if (!output.CanWrite) + throw new ArgumentException("IPSW creation requires a writable output stream.", nameof(output)); + + if (output.CanSeek) { + output.Position = 0; + output.SetLength(0); } - zip.Finish(); - } - // ── IArchiveModifiable ───────────────────────────────────────────── + using var zip = new ZipArchive(output, ZipArchiveMode.Create, leaveOpen: true); + foreach (var input in inputs) { + var name = NormalizeZipPath(input.ArchiveName); + if (IsSynthetic(name)) + continue; + + if (input.IsDirectory) { + if (!name.EndsWith('/')) + name += '/'; + zip.CreateEntry(name, CompressionLevel.NoCompression); + continue; + } + + var size = input.InMemoryContent?.LongLength ?? new FileInfo(input.FullPath).Length; + var entry = zip.CreateEntry(name, SelectCompression(name, size)); + TryApplyTimestamp(entry, input); + using var destination = entry.Open(); + if (input.InMemoryContent is { } bytes) { + destination.Write(bytes); + } else { + using var source = File.OpenRead(input.FullPath); + source.CopyTo(destination); + } + } + } - /// - /// Adds (or replaces by ZIP path) entries inside an existing IPSW. - /// Routes through — only the central - /// directory, EOCD, and the appended LFH + payload are touched. - /// Synthetic canonical entries are silently dropped. - /// public void Add(Stream archive, IReadOnlyList inputs) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(inputs); - foreach (var (name, data) in FilesOnly(inputs)) { - if (string.Equals(name, "FULL.ipsw", StringComparison.OrdinalIgnoreCase)) continue; - if (string.Equals(name, "metadata.ini", StringComparison.OrdinalIgnoreCase)) continue; - IpswInPlaceModifier.AddEntry(archive, name, data); + foreach (var input in inputs) { + if (input.IsDirectory) + continue; + var name = NormalizeZipPath(input.ArchiveName); + if (IsSynthetic(name)) + continue; + IpswInPlaceModifier.AddEntry(archive, name, input.ReadContent()); } } - /// - /// Removes named ZIP entries from an existing IPSW. Routes through - /// — the LFH + compressed payload of - /// the dropped entry are zero-wiped and the central directory is rewritten. - /// public void Remove(Stream archive, string[] entryNames) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(entryNames); foreach (var name in entryNames) { - if (string.Equals(name, "FULL.ipsw", StringComparison.OrdinalIgnoreCase)) continue; - if (string.Equals(name, "metadata.ini", StringComparison.OrdinalIgnoreCase)) continue; - IpswInPlaceModifier.RemoveEntry(archive, name); + var normalized = NormalizeZipPath(name); + if (IsSynthetic(normalized)) + continue; + IpswInPlaceModifier.RemoveEntry(archive, normalized); } } - private static bool Wants(string[]? files, string name) - => files == null || files.Length == 0 || MatchesFilter(name, files); + public void Defragment(Stream archive) + => this.Defragment(archive, new DefragOptions { Mode = DefragMode.ConsolidateAtStart }); - private static (List Canonical, int TotalZipEntries) EnumerateCanonical(Stream stream) { - stream.Seek(0, SeekOrigin.Begin); - using var zip = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); - return EnumerateCanonicalFromZip(zip); + public void Defragment(Stream archive, DefragOptions options) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(options); + if (options.Mode != DefragMode.ConsolidateAtStart) + throw new NotSupportedException($"IPSW defrag supports only {DefragMode.ConsolidateAtStart}."); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("IPSW defrag requires a readable, writable, seekable stream.", nameof(archive)); + + using var rebuilt = CreateScratchStream(); + RebuildVerb.RebuildToStream( + archive, + rebuilt, + this, + this, + syntheticNames: SyntheticEntries, + onProgress: options.OnProgress, + cancellationToken: options.CancellationToken); + + options.CancellationToken.ThrowIfCancellationRequested(); + options.OnProgress?.Invoke(new DefragProgressEvent( + "committing", 0.99, -1, 0, Math.Max(1, rebuilt.Length), null, + "Committing verified IPSW rebuild")); + + archive.Position = 0; + archive.SetLength(0); + rebuilt.Position = 0; + rebuilt.CopyTo(archive); + archive.Flush(); + + archive.Position = 0; + var finalLayout = this.EnumerateLayout(archive).ToArray(); + options.OnProgress?.Invoke(new DefragProgressEvent( + "complete", 1, -1, -1, Math.Max(1, archive.Length), finalLayout, + "IPSW rebuild committed successfully")); } - private static (List Canonical, int TotalZipEntries) EnumerateCanonicalFromZip(ZipArchive zip) { - var canonical = new List(); - foreach (var entry in zip.Entries) { - // Skip directory entries (trailing slash, zero length implied). - if (entry.FullName.EndsWith('/')) continue; + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("IPSW shrink requires a readable, seekable input stream.", nameof(input)); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("IPSW shrink requires a writable, seekable output stream.", nameof(output)); - var name = entry.FullName.Replace('\\', '/'); - var filename = Path.GetFileName(name); - var method = entry.CompressedLength == entry.Length ? "Stored" : "Deflate"; - DateTime? lastModified = entry.LastWriteTime.DateTime; - - string canonicalName; - string? kind; - if (string.Equals(filename, "BuildManifest.plist", StringComparison.OrdinalIgnoreCase)) { - canonicalName = "BuildManifest.plist"; - kind = "manifest"; - } else if (name.Contains("Firmware/", StringComparison.OrdinalIgnoreCase)) { - canonicalName = "Firmware/" + filename; - kind = "firmware"; - } else if (IsBootloaderStage(filename)) { - canonicalName = filename; - kind = "bootloader"; - } else if (filename.EndsWith(".dmg", StringComparison.OrdinalIgnoreCase)) { - canonicalName = filename; - kind = "rootfs"; - } else { - canonicalName = "other/" + name; - kind = "other"; + using var rebuilt = CreateScratchStream(); + var useRebuilt = false; + try { + RebuildVerb.RebuildToStream(input, rebuilt, this, this, syntheticNames: SyntheticEntries); + useRebuilt = rebuilt.Length > 0 && rebuilt.Length < input.Length; + } catch { + useRebuilt = false; + } + + if (ReferenceEquals(input, output)) { + if (!useRebuilt) { + input.Position = 0; + return; } + output.Position = 0; + output.SetLength(0); + rebuilt.Position = 0; + rebuilt.CopyTo(output); + output.Flush(); + output.Position = 0; + return; + } + + output.Position = 0; + output.SetLength(0); + if (useRebuilt) { + rebuilt.Position = 0; + rebuilt.CopyTo(output); + } else { + input.Position = 0; + input.CopyTo(output); + } + output.Flush(); + output.Position = 0; + } + + private static List EnumerateEntries(Stream stream) { + stream.Position = 0; + using var zip = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); + var result = new List(zip.Entries.Count); + foreach (var entry in zip.Entries) { + var name = NormalizeZipPath(entry.FullName); + var isDirectory = IsDirectoryEntry(entry); + var method = entry.CompressedLength == entry.Length ? "Stored" : "Deflate"; + result.Add(new IpswEntry( + name, + entry.Length, + entry.CompressedLength, + method, + isDirectory, + entry.LastWriteTime.DateTime, + ClassifyKind(name, isDirectory))); + } + return result; + } + + private static byte[] BuildMetadata(ZipArchive zip) { + string? identifier = null; + string? productVersion = null; + string? buildVersion = null; + var manifest = zip.Entries.FirstOrDefault(entry => + string.Equals(NormalizeZipPath(entry.FullName), "BuildManifest.plist", StringComparison.OrdinalIgnoreCase)); + if (manifest != null && manifest.Length <= 16 * 1024 * 1024) { + using var source = manifest.Open(); + using var memory = new MemoryStream(); + source.CopyTo(memory); + TryParsePlistFields(memory.ToArray(), out identifier, out productVersion, out buildVersion); + } + return Encoding.UTF8.GetBytes(BuildMetadataIni(identifier, productVersion, buildVersion, zip.Entries.Count)); + } + + private static string ClassifyKind(string name, bool isDirectory) { + if (isDirectory) + return "directory"; + var fileName = LeafName(name); + if (string.Equals(name, "BuildManifest.plist", StringComparison.OrdinalIgnoreCase)) + return "manifest"; + if (string.Equals(name, "Restore.plist", StringComparison.OrdinalIgnoreCase) || + string.Equals(name, "RestoreVersion.plist", StringComparison.OrdinalIgnoreCase)) + return "restore-metadata"; + if (name.StartsWith("Firmware/", StringComparison.OrdinalIgnoreCase)) + return "firmware"; + if (IsBootloaderStage(fileName)) + return "bootloader"; + if (fileName.EndsWith(".dmg", StringComparison.OrdinalIgnoreCase)) + return "disk-image"; + return "other"; + } + + private static CompressionLevel SelectCompression(string name, long size) { + var fileName = LeafName(name); + if (size >= 32L * 1024 * 1024 || + fileName.EndsWith(".dmg", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".img4", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".im4p", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".im4m", StringComparison.OrdinalIgnoreCase)) + return CompressionLevel.NoCompression; + return CompressionLevel.Optimal; + } - // Use entry.Length — never call entry.Open() during List enumeration. - canonical.Add(new CanonicalEntry( - CanonicalName: canonicalName, - ZipEntryName: entry.FullName, - Size: entry.Length, - Method: method, - LastModified: lastModified, - Kind: kind)); + private static void TryApplyTimestamp(ZipArchiveEntry entry, ArchiveInputInfo input) { + if (input.InMemoryContent != null || !File.Exists(input.FullPath)) + return; + try { + entry.LastWriteTime = File.GetLastWriteTime(input.FullPath); + } catch (ArgumentOutOfRangeException) { + // ZIP/DOS timestamps are bounded to 1980..2107; payload data is unaffected. } - var total = zip.Entries.Count; - return (canonical, total); } - private static bool IsBootloaderStage(string filename) { - // Apple boot stage prefixes: LLB., iBSS., iBEC., iBoot. - if (filename.StartsWith("LLB.", StringComparison.OrdinalIgnoreCase)) return true; - if (filename.StartsWith("iBSS.", StringComparison.OrdinalIgnoreCase)) return true; - if (filename.StartsWith("iBEC.", StringComparison.OrdinalIgnoreCase)) return true; - if (filename.StartsWith("iBoot.", StringComparison.OrdinalIgnoreCase)) return true; - return false; + private static bool Wants(string[]? files, string name) + => files == null || files.Length == 0 || MatchesFilter(name, files); + + private static bool IsSynthetic(string name) + => SyntheticEntries.Contains(name); + + private static bool IsDirectoryEntry(ZipArchiveEntry entry) + => entry.FullName.EndsWith('/') || entry.FullName.EndsWith('\\'); + + private static string NormalizeZipPath(string name) + => name.Replace('\\', '/').TrimStart('/'); + + private static string LeafName(string name) { + var normalized = NormalizeZipPath(name).TrimEnd('/'); + var slash = normalized.LastIndexOf('/'); + return slash < 0 ? normalized : normalized[(slash + 1)..]; } - /// - /// Mirrors the path-sanitization done by so we can - /// stream directly into the target file without first materializing the payload. - /// + private static bool IsBootloaderStage(string filename) + => filename.StartsWith("LLB.", StringComparison.OrdinalIgnoreCase) || + filename.StartsWith("iBSS.", StringComparison.OrdinalIgnoreCase) || + filename.StartsWith("iBEC.", StringComparison.OrdinalIgnoreCase) || + filename.StartsWith("iBoot.", StringComparison.OrdinalIgnoreCase); + private static string SafeCombine(string baseDir, string entryName) { - var safeName = entryName.Replace('\\', '/').TrimStart('/'); - if (safeName.Contains("..")) safeName = Path.GetFileName(safeName); - return Path.Combine(baseDir, safeName); + var normalized = NormalizeZipPath(entryName); + var segments = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Any(segment => segment is "." or "..")) + throw new InvalidDataException($"IPSW entry path escapes extraction root: {entryName}"); + + var root = Path.GetFullPath(baseDir); + var candidate = Path.GetFullPath(Path.Combine(root, Path.Combine(segments))); + var rootPrefix = root.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; + if (!candidate.StartsWith(rootPrefix, StringComparison.OrdinalIgnoreCase) && + !string.Equals(candidate, root, StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException($"IPSW entry path escapes extraction root: {entryName}"); + return candidate; } - /// - /// Best-effort plist field scrape. We intentionally avoid full plist parsing; this just finds - /// the <key>...</key><string>...</string> pairs for a few known Apple - /// manifest fields (ProductVersion, ProductBuildVersion, identifier-like). Returns null if the - /// plist is binary or un-parseable. - /// private static void TryParsePlistFields(byte[] data, out string? identifier, out string? productVersion, out string? buildVersion) { identifier = null; productVersion = null; buildVersion = null; - if (data.Length < 16) return; - // Binary plist — give up; we surface raw bytes so consumers can parse. - if (data[0] == (byte)'b' && data[1] == (byte)'p' && data[2] == (byte)'l') return; - - string text; - try { text = Encoding.UTF8.GetString(data); } - catch { return; } + if (data.Length < 16) + return; + if (data.AsSpan().StartsWith("bplist"u8)) + return; + var text = Encoding.UTF8.GetString(data); productVersion = FindStringValue(text, "ProductVersion"); buildVersion = FindStringValue(text, "ProductBuildVersion") ?? FindStringValue(text, "BuildVersion"); identifier = FindStringValue(text, "ProductType") ?? FindStringValue(text, "Identifier"); @@ -320,23 +437,55 @@ private static void TryParsePlistFields(byte[] data, out string? identifier, out private static string? FindStringValue(string plistXml, string key) { var keyTag = $"{key}"; - var keyIdx = plistXml.IndexOf(keyTag, StringComparison.Ordinal); - if (keyIdx < 0) return null; - var searchFrom = keyIdx + keyTag.Length; - var openIdx = plistXml.IndexOf("", searchFrom, StringComparison.Ordinal); - if (openIdx < 0) return null; - var closeIdx = plistXml.IndexOf("", openIdx + 8, StringComparison.Ordinal); - if (closeIdx < 0) return null; - return plistXml.Substring(openIdx + 8, closeIdx - (openIdx + 8)); + var keyIndex = plistXml.IndexOf(keyTag, StringComparison.Ordinal); + if (keyIndex < 0) + return null; + var openIndex = plistXml.IndexOf("", keyIndex + keyTag.Length, StringComparison.Ordinal); + if (openIndex < 0) + return null; + var closeIndex = plistXml.IndexOf("", openIndex + 8, StringComparison.Ordinal); + return closeIndex < 0 ? null : plistXml[(openIndex + 8)..closeIndex]; } private static string BuildMetadataIni(string? identifier, string? productVersion, string? buildVersion, int totalZipEntries) { - var sb = new StringBuilder(); - sb.Append("[Ipsw]\n"); - sb.Append(CultureInfo.InvariantCulture, $"identifier={identifier ?? string.Empty}\n"); - sb.Append(CultureInfo.InvariantCulture, $"product_version={productVersion ?? string.Empty}\n"); - sb.Append(CultureInfo.InvariantCulture, $"build_version={buildVersion ?? string.Empty}\n"); - sb.Append(CultureInfo.InvariantCulture, $"total_zip_entries={totalZipEntries}\n"); - return sb.ToString(); + var builder = new StringBuilder(); + builder.Append("[Ipsw]\n"); + builder.Append(CultureInfo.InvariantCulture, $"identifier={identifier ?? string.Empty}\n"); + builder.Append(CultureInfo.InvariantCulture, $"product_version={productVersion ?? string.Empty}\n"); + builder.Append(CultureInfo.InvariantCulture, $"build_version={buildVersion ?? string.Empty}\n"); + builder.Append(CultureInfo.InvariantCulture, $"total_zip_entries={totalZipEntries}\n"); + return builder.ToString(); + } + + private static FileStream CreateScratchStream() + => new( + Path.Combine(Path.GetTempPath(), "cwb_ipsw_" + Guid.NewGuid().ToString("N") + ".tmp"), + FileMode.CreateNew, + FileAccess.ReadWrite, + FileShare.None, + 64 * 1024, + FileOptions.DeleteOnClose); + + private sealed class ZipOwnedReadStream(Stream inner, ZipArchive owner) : Stream { + public override bool CanRead => inner.CanRead; + public override bool CanSeek => inner.CanSeek; + public override bool CanWrite => false; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => inner.Position = value; } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); + public override int Read(Span buffer) => inner.Read(buffer); + public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override void Write(ReadOnlySpan buffer) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) { + if (disposing) { + inner.Dispose(); + owner.Dispose(); + } + base.Dispose(disposing); + } } } diff --git a/FileFormats/FileFormat.Zip/ZipLayoutMap.cs b/FileFormats/FileFormat.Zip/ZipLayoutMap.cs index a4866626e..bf55fd6ef 100644 --- a/FileFormats/FileFormat.Zip/ZipLayoutMap.cs +++ b/FileFormats/FileFormat.Zip/ZipLayoutMap.cs @@ -1,23 +1,34 @@ #pragma warning disable CS1591 +using System.Buffers.Binary; using Compression.Registry; namespace FileFormat.Zip; /// -/// Walks the ZIP central directory and emits the byte-level layout of every -/// local file header, compressed data payload, the central directory itself, -/// and the EOCD record as tiles. +/// Walks a ZIP central directory and emits a fail-closed byte-level layout of local headers, +/// compressed payloads, optional data descriptors, the central directory and EOCD. /// public static class ZipLayoutMap { - /// - /// Enumerates the value. - /// + private sealed record EntryLayout( + ZipEntry Entry, + long HeaderOffset, + int HeaderLength, + long DataOffset, + long DataLength, + int DescriptorLength) { + public long End => this.DataOffset + this.DataLength + this.DescriptorLength; + } + public static IEnumerable Enumerate(Stream archive) { ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanSeek) + yield break; + archive.Position = 0; - long cdOffset, cdSize; + long cdOffset; + long cdSize; int cdCount; long eocdOffset; try { @@ -27,53 +38,69 @@ public static IEnumerable Enumerate(Stream archive) { yield break; } - // Read central directory entries to learn local header offsets - archive.Position = cdOffset; - var reader = new BinaryReader(archive, System.Text.Encoding.Latin1, leaveOpen: true); - var entries = new List(); - for (var i = 0; i < cdCount; i++) { - try { - entries.Add(ZipCentralDirectoryEntry.Read(reader)); - } catch { - break; - } + List entries; + try { + entries = ReadCentralDirectory(archive, cdOffset, cdCount); + } catch { + yield break; } - // For each entry, emit local file header (MetadataReserved) + compressed data (Used) - foreach (var entry in entries) { - // Determine LFH size: fixed 30 bytes + filename + extra field - archive.Position = entry.LocalHeaderOffset; - if (archive.Position + 30 > archive.Length) + var ordered = entries.OrderBy(entry => entry.LocalHeaderOffset).ToArray(); + var layouts = new List(ordered.Length); + for (var i = 0; i < ordered.Length; ++i) { + var entry = ordered[i]; + if (!TryReadLocalHeaderLength(archive, entry.LocalHeaderOffset, out var headerLength)) continue; - var lfhBuf = new byte[30]; - if (archive.Read(lfhBuf, 0, 30) < 30) + var dataOffset = entry.LocalHeaderOffset + headerLength; + if (dataOffset < 0 || entry.CompressedSize < 0 || dataOffset > archive.Length - entry.CompressedSize) continue; - var fnLen = BitConverter.ToUInt16(lfhBuf, 26); - var exLen = BitConverter.ToUInt16(lfhBuf, 28); - var lfhSize = 30 + fnLen + exLen; + var dataEnd = dataOffset + entry.CompressedSize; + var nextBoundary = cdOffset; + for (var next = i + 1; next < ordered.Length; ++next) { + if (ordered[next].LocalHeaderOffset > entry.LocalHeaderOffset) { + nextBoundary = Math.Min(nextBoundary, ordered[next].LocalHeaderOffset); + break; + } + } + nextBoundary = Math.Clamp(nextBoundary, dataEnd, archive.Length); - // MetadataReserved tile for the local file header - yield return new DefragBlockInfo( + var descriptorLength = ReadDataDescriptorLength(archive, entry, dataEnd, nextBoundary); + layouts.Add(new EntryLayout( + entry, entry.LocalHeaderOffset, - lfhSize, + headerLength, + dataOffset, + entry.CompressedSize, + descriptorLength)); + } + + foreach (var layout in layouts) { + yield return new DefragBlockInfo( + layout.HeaderOffset, + layout.HeaderLength, DefragBlockKind.MetadataReserved, - FileName: $"LFH: {entry.FileName}"); + FileName: $"LFH: {layout.Entry.FileName}"); - // Used tile for compressed data - if (entry.CompressedSize > 0) { - var classification = ClassifyMethod(entry.CompressionMethod); + if (layout.DataLength > 0) { yield return new DefragBlockInfo( - entry.LocalHeaderOffset + lfhSize, - entry.CompressedSize, + layout.DataOffset, + layout.DataLength, DefragBlockKind.Used, - FileName: entry.FileName, - Classification: classification); + FileName: layout.Entry.FileName, + Classification: ClassifyMethod(layout.Entry.CompressionMethod)); + } + + if (layout.DescriptorLength > 0) { + yield return new DefragBlockInfo( + layout.DataOffset + layout.DataLength, + layout.DescriptorLength, + DefragBlockKind.MetadataReserved, + FileName: $"Data descriptor: {layout.Entry.FileName}"); } } - // Central directory = one MetadataReserved tile if (cdSize > 0) { yield return new DefragBlockInfo( cdOffset, @@ -82,33 +109,21 @@ public static IEnumerable Enumerate(Stream archive) { FileName: "Central Directory"); } - // EOCD = one MetadataReserved tile (from EOCD start to end of file) if (eocdOffset >= 0) { - var eocdSize = archive.Length - eocdOffset; yield return new DefragBlockInfo( eocdOffset, - eocdSize, + archive.Length - eocdOffset, DefragBlockKind.MetadataReserved, FileName: "End of Central Directory"); } - // Detect gaps (free/dead bytes) between entries - var regions = new List<(long Start, long End)>(); - foreach (var entry in entries) { - archive.Position = entry.LocalHeaderOffset; - if (archive.Position + 30 > archive.Length) continue; - var buf = new byte[30]; - if (archive.Read(buf, 0, 30) < 30) continue; - var fn = BitConverter.ToUInt16(buf, 26); - var ex = BitConverter.ToUInt16(buf, 28); - var totalEntrySize = 30 + fn + ex + entry.CompressedSize; - regions.Add((entry.LocalHeaderOffset, entry.LocalHeaderOffset + totalEntrySize)); - } - regions.Add((cdOffset, cdOffset + cdSize)); + var regions = new List<(long Start, long End)>(layouts.Count + 2); + regions.AddRange(layouts.Select(layout => (layout.HeaderOffset, layout.End))); + if (cdSize > 0) + regions.Add((cdOffset, cdOffset + cdSize)); if (eocdOffset >= 0) regions.Add((eocdOffset, archive.Length)); - - regions.Sort((a, b) => a.Start.CompareTo(b.Start)); + regions.Sort(static (left, right) => left.Start.CompareTo(right.Start)); var cursor = 0L; foreach (var (start, end) in regions) { @@ -119,35 +134,138 @@ public static IEnumerable Enumerate(Stream archive) { DefragBlockKind.Free, FileName: "Dead space"); } - if (end > cursor) cursor = end; + cursor = Math.Max(cursor, end); + } + + if (cursor < archive.Length) { + yield return new DefragBlockInfo( + cursor, + archive.Length - cursor, + DefragBlockKind.Free, + FileName: "Dead space"); } } + private static List ReadCentralDirectory(Stream archive, long cdOffset, int cdCount) { + archive.Position = cdOffset; + using var reader = new BinaryReader(archive, System.Text.Encoding.Latin1, leaveOpen: true); + var entries = new List(cdCount); + for (var i = 0; i < cdCount; ++i) + entries.Add(ZipCentralDirectoryEntry.Read(reader)); + return entries; + } + + private static bool TryReadLocalHeaderLength(Stream archive, long offset, out int length) { + length = 0; + if (offset < 0 || offset > archive.Length - 30) + return false; + + Span header = stackalloc byte[30]; + archive.Position = offset; + if (!ReadExactly(archive, header)) + return false; + if (BinaryPrimitives.ReadUInt32LittleEndian(header) != ZipConstants.LocalFileHeaderSignature) + return false; + + var fileNameLength = BinaryPrimitives.ReadUInt16LittleEndian(header[26..]); + var extraLength = BinaryPrimitives.ReadUInt16LittleEndian(header[28..]); + length = 30 + fileNameLength + extraLength; + return offset <= archive.Length - length; + } + + /// + /// Returns the number of bytes occupied by an optional ZIP data descriptor. When bit 3 says a + /// descriptor exists but its values cannot be proven, the maximum standard descriptor span up + /// to the next known structure is reserved rather than exposed as free space. + /// + private static int ReadDataDescriptorLength(Stream archive, ZipEntry entry, long offset, long nextBoundary) { + if ((entry.GeneralPurposeFlags & ZipConstants.FlagDataDescriptor) == 0) + return 0; + + var availableLong = Math.Min(24L, Math.Max(0, nextBoundary - offset)); + var available = (int)availableLong; + if (available == 0) + return 0; + + Span data = stackalloc byte[24]; + archive.Position = offset; + var read = ReadUpTo(archive, data[..available]); + if (read == 0) + return 0; + data = data[..read]; + + var signaturePresent = data.Length >= 4 && + BinaryPrimitives.ReadUInt32LittleEndian(data) == ZipConstants.DataDescriptorSignature; + + if (signaturePresent) { + if (Matches32BitDescriptor(data, 4, entry)) + return 16; + if (Matches64BitDescriptor(data, 4, entry)) + return 24; + } + + if (Matches32BitDescriptor(data, 0, entry)) + return 12; + if (Matches64BitDescriptor(data, 0, entry)) + return 20; + + return read; + } + + private static bool Matches32BitDescriptor(ReadOnlySpan data, int offset, ZipEntry entry) { + if (entry.CompressedSize > uint.MaxValue || entry.UncompressedSize > uint.MaxValue || data.Length < offset + 12) + return false; + return BinaryPrimitives.ReadUInt32LittleEndian(data[offset..]) == entry.Crc32 && + BinaryPrimitives.ReadUInt32LittleEndian(data[(offset + 4)..]) == (uint)entry.CompressedSize && + BinaryPrimitives.ReadUInt32LittleEndian(data[(offset + 8)..]) == (uint)entry.UncompressedSize; + } + + private static bool Matches64BitDescriptor(ReadOnlySpan data, int offset, ZipEntry entry) { + if (data.Length < offset + 20) + return false; + return BinaryPrimitives.ReadUInt32LittleEndian(data[offset..]) == entry.Crc32 && + BinaryPrimitives.ReadUInt64LittleEndian(data[(offset + 4)..]) == (ulong)entry.CompressedSize && + BinaryPrimitives.ReadUInt64LittleEndian(data[(offset + 12)..]) == (ulong)entry.UncompressedSize; + } + + private static bool ReadExactly(Stream stream, Span buffer) { + var total = 0; + while (total < buffer.Length) { + var read = stream.Read(buffer[total..]); + if (read <= 0) + return false; + total += read; + } + return true; + } + + private static int ReadUpTo(Stream stream, Span buffer) { + var total = 0; + while (total < buffer.Length) { + var read = stream.Read(buffer[total..]); + if (read <= 0) + break; + total += read; + } + return total; + } + private static DefragBlockClass ClassifyMethod(ZipCompressionMethod method) => method switch { ZipCompressionMethod.Store => DefragBlockClass.Frozen, - ZipCompressionMethod.Deflate => DefragBlockClass.Normal, - ZipCompressionMethod.Deflate64 => DefragBlockClass.Normal, + ZipCompressionMethod.Deflate or ZipCompressionMethod.Deflate64 => DefragBlockClass.Normal, ZipCompressionMethod.BZip2 => DefragBlockClass.Cold, - ZipCompressionMethod.Lzma => DefragBlockClass.Hot, - ZipCompressionMethod.Zstd => DefragBlockClass.Hot, - ZipCompressionMethod.Ppmd => DefragBlockClass.Hot, + ZipCompressionMethod.Lzma or ZipCompressionMethod.Zstd or ZipCompressionMethod.Ppmd => DefragBlockClass.Hot, _ => DefragBlockClass.Normal, }; private static long FindEocdOffset(Stream stream) { - var searchLen = Math.Min(stream.Length, 65557); - var searchStart = stream.Length - searchLen; - var buffer = new byte[searchLen]; + var searchLength = Math.Min(stream.Length, 65_557); + var searchStart = stream.Length - searchLength; + var buffer = new byte[checked((int)searchLength)]; stream.Position = searchStart; - var bytesRead = 0; - while (bytesRead < buffer.Length) { - var read = stream.Read(buffer, bytesRead, buffer.Length - bytesRead); - if (read == 0) break; - bytesRead += read; - } + var bytesRead = ReadUpTo(stream, buffer); for (var i = bytesRead - 22; i >= 0; --i) { - if (buffer[i] == 0x50 && buffer[i + 1] == 0x4B && - buffer[i + 2] == 0x05 && buffer[i + 3] == 0x06) + if (BinaryPrimitives.ReadUInt32LittleEndian(buffer.AsSpan(i, 4)) == ZipConstants.EndOfCentralDirectorySignature) return searchStart + i; } return -1; From 32098de814a80ca16c1f29e64b37d3c1eb90b975 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:05:14 +0200 Subject: [PATCH 064/165] Temporarily patch generated Lynx capability ledgers --- .github/workflows/patch-lynx-ledgers.yml | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/patch-lynx-ledgers.yml diff --git a/.github/workflows/patch-lynx-ledgers.yml b/.github/workflows/patch-lynx-ledgers.yml new file mode 100644 index 000000000..f804819e3 --- /dev/null +++ b/.github/workflows/patch-lynx-ledgers.yml @@ -0,0 +1,50 @@ +name: Patch Lynx ledgers + +on: + push: + branches: + - feature/commodore-lynx-maintenance + +permissions: + contents: write + +jobs: + patch: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: feature/commodore-lynx-maintenance + + - name: Update generated capability rows + shell: python + run: | + from pathlib import Path + + replacements = { + Path("Hawkynt.FileFormats.Archives/README.md"): ( + "| Lynx (Commodore) | `Lynx` | `.lnx` | R/W | ✅ | defrag · wipe | Stored entries only | [Archive Team](http://fileformats.archiveteam.org/wiki/Lynx_(Commodore_64)) |", + "| Lynx (Commodore) | `Lynx` | `.lnx` | R/W | ✅ | defrag · shrink · wipe | Stored entries only | [Archive Team](http://fileformats.archiveteam.org/wiki/Lynx_(Commodore_64)) |", + ), + Path("Hawkynt.FileFormats.FileSystems/README.md"): ( + "| [Commodore Lynx archive](https://en.wikipedia.org/wiki/Commodore_64) | `Lynx` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 Lynx archive | [Lynx](https://ist.uwaterloo.ca/~schepers/formats/LNX.TXT) |", + "| [Commodore Lynx archive](https://en.wikipedia.org/wiki/Commodore_64) | `Lynx` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | C64 Lynx archive | [Lynx](https://ist.uwaterloo.ca/~schepers/formats/LNX.TXT) |", + ), + } + + for path, (old, new) in replacements.items(): + text = path.read_text(encoding="utf-8") + if text.count(old) != 1: + raise SystemExit(f"Expected exactly one current Lynx row in {path}") + path.write_text(text.replace(old, new), encoding="utf-8", newline="\n") + + - name: Commit capability ledgers + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add Hawkynt.FileFormats.Archives/README.md Hawkynt.FileFormats.FileSystems/README.md + git diff --cached --quiet && exit 0 + git commit -m "Update Lynx capability ledgers" + git push origin HEAD:feature/commodore-lynx-maintenance From f5c4c67ad2da4a9d7951dcc025ad37ac1ec0369c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:05:23 +0200 Subject: [PATCH 065/165] Add safe MooseFS metadata maintenance operations --- .../MooseFsFormatDescriptor.cs | 198 +++++++++++------- 1 file changed, 127 insertions(+), 71 deletions(-) diff --git a/FileSystems/FileSystem.MooseFs/MooseFsFormatDescriptor.cs b/FileSystems/FileSystem.MooseFs/MooseFsFormatDescriptor.cs index cf1155581..3a0086381 100644 --- a/FileSystems/FileSystem.MooseFs/MooseFsFormatDescriptor.cs +++ b/FileSystems/FileSystem.MooseFs/MooseFsFormatDescriptor.cs @@ -6,115 +6,171 @@ namespace FileSystem.MooseFs; /// -/// Partial R/O descriptor for MooseFS master-metadata images -/// (metadata.mfs). Surfaces the metadata envelope (signature, -/// counters, section index) and the raw payload bytes of each walked -/// section. Path-tree (NODE/EDGE) and chunk-id (CHNK) bodies are -/// version-specific and require golden samples to decode honestly — the -/// reader makes no claim about their internal structure. +/// Descriptor for MooseFS master-metadata images (metadata.mfs). +/// The standalone image contains namespace/chunk metadata, not file payloads; +/// payload bytes live on chunk servers. The descriptor therefore exposes a +/// synthetic forensic view of the metadata envelope rather than pretending a +/// metadata dump is a self-contained MooseFS volume. /// /// -/// MooseFS file content lives on chunk servers and is unreachable from a -/// single metadata image. Listing therefore surfaces ONLY synthetic -/// metadata + per-section raw payloads, never POSIX paths. +/// Safe standalone-image maintenance is deliberately narrow: byte layout can +/// be mapped, unused-space wiping fails closed (valid metadata dumps are tightly +/// packed), and purge resets the image to MooseFS's official MFSM NEW +/// empty bootstrap. Add/replace/remove, shrink and defragmentation require +/// semantic NODE/EDGE/CHNK updates and/or live cluster coordination and are not +/// advertised here. /// /// /// References: /// -/// https://github.com/moosefs/moosefs — canonical source (master metadata dump/load code) -/// https://moosefs.com/ — vendor site and documentation -/// https://en.wikipedia.org/wiki/Moose_File_System — Wikipedia article +/// https://github.com/moosefs/moosefs — canonical implementation and metadata loader/store format +/// https://moosefs.com/ — vendor documentation /// /// -public sealed class MooseFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { +public sealed class MooseFsFormatDescriptor + : IFormatDescriptor, IArchiveFormatOperations, IArchiveLayoutMap, IArchivePurgeable { - /// - /// Gets the id. - /// + /// Gets the id. public string Id => "MooseFs"; - /// - /// Gets the display name. - /// + + /// Gets the display name. public string DisplayName => "MooseFS"; - /// - /// Gets the category. - /// + + /// Gets the category. public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// + + /// Gets the capabilities. public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest; - /// - /// Gets the default extension. - /// + + /// Gets the default extension. public string DefaultExtension => ".mfsm"; - // Note: ".mfs" intentionally NOT claimed — collides with Macintosh File System (FileSystem.Mfs). - // MooseFS detection is by magic 'MFSM' at offset 0; the ".mfsm" extension is MooseFS-specific. - /// - /// Gets the extensions. - /// + + // ".mfs" intentionally is not claimed: it collides with Macintosh File System (FileSystem.Mfs). + // MooseFS detection is by the MFSM signature at offset zero. + /// Gets the extensions. public IReadOnlyList Extensions => [".mfsm"]; - /// - /// Gets the compound extensions. - /// + + /// Gets the compound extensions. public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// + + /// Gets the magic signatures. public IReadOnlyList MagicSignatures => [ - // ASCII "MFSM" at offset 0 — MooseFS master metadata tag. new("MFSM"u8.ToArray(), Offset: 0, Confidence: 0.90), ]; - /// - /// Gets the methods. - /// + + /// Gets the methods. public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// + + /// Gets the tar compression format id. public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// + + /// Gets the family. public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// + + /// Gets the description. public string Description => - "MooseFS — partial R/O of master metadata envelope (signature + section " + - "index). File content lives on chunk servers and is unreachable from a " + - "single metadata image; only metadata.ini + raw image + per-section " + - "raw payloads are surfaced."; + "MooseFS master metadata image. Reads the versioned metadata envelope and section table; " + + "file payloads remain on chunk servers. Standalone maintenance provides byte-layout mapping, " + + "safe unused-space wiping and destructive reset to the official empty MFSM NEW bootstrap; " + + "full namespace R/W, shrink and defragmentation are intentionally not claimed."; + + /// Lists the synthetic entries in the supplied metadata image. + public List List(Stream stream, string? password) { + using var reader = new MooseFsReader(stream); + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo( + index, entry.Name, entry.Size, entry.Size, "Stored", entry.IsDirectory, false, null)).ToList(); + } + + /// Extracts the synthetic metadata-image views. + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + using var reader = new MooseFsReader(stream); + foreach (var entry in reader.Entries) { + if (entry.IsDirectory) + continue; + if (files != null && !MatchesFilter(entry.Name, files)) + continue; + WriteFile(outputDir, entry.Name, reader.Extract(entry)); + } + } /// - /// Lists the entries in the supplied container. + /// Enumerates the exact bytes occupied by a valid MooseFS metadata image. + /// Unknown/truncated images return no extents so generic maintenance fails closed. /// - public List List(Stream stream, string? password) { - var r = new MooseFsReader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo( - i, e.Name, e.Size, e.Size, "Stored", e.IsDirectory, false, null)).ToList(); + public IEnumerable EnumerateLayout(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanSeek) + return []; + + var originalPosition = archive.Position; + try { + archive.Position = 0; + using var reader = new MooseFsReader(archive); + if (reader.ParseStatus != "ok") + return []; + + if (reader.IsEmptyBootstrap) + return [new DefragBlockInfo(0, reader.ImageSize, DefragBlockKind.MetadataReserved, "MFSM NEW bootstrap")]; + + if (reader.FileFormatVersion is byte version && version < 0x16) + return [new DefragBlockInfo(0, reader.ImageSize, DefragBlockKind.MetadataReserved, "legacy MooseFS metadata")]; + + var result = new List { + new(0, 24, DefragBlockKind.MetadataReserved, "MooseFS metadata header"), + }; + + foreach (var section in reader.Sections) { + result.Add(new DefragBlockInfo( + section.Offset - 16, 16, DefragBlockKind.MetadataReserved, $"{section.Tag} section header")); + if (section.Length > 0) + result.Add(new DefragBlockInfo(section.Offset, section.Length, DefragBlockKind.Used, section.Tag)); + } + + result.Add(new DefragBlockInfo( + reader.ImageSize - MooseFsReader.EofMarker.Length, + MooseFsReader.EofMarker.Length, + DefragBlockKind.MetadataReserved, + "MooseFS EOF marker")); + return result; + } catch (InvalidDataException) { + return []; + } finally { + archive.Position = originalPosition; + } } /// - /// Decodes the supplied input. + /// Resets a valid metadata image to MooseFS's official empty bootstrap state. + /// This only rewrites metadata.mfs; any chunk-server data is outside the + /// image and therefore outside this operation. /// - public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new MooseFsReader(stream); - foreach (var e in r.Entries) { - if (e.IsDirectory) continue; - if (files != null && !MatchesFilter(e.Name, files)) continue; - WriteFile(outputDir, e.Name, r.Extract(e)); + public void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException( + "MooseFS purge requires a readable, writable, seekable stream.", nameof(archive)); + + archive.Position = 0; + using (var reader = new MooseFsReader(archive)) { + if (reader.ParseStatus != "ok") + throw new InvalidDataException( + $"MooseFS purge requires a structurally valid metadata image; parse status is '{reader.ParseStatus}'."); } + + archive.Position = 0; + archive.Write("MFSM NEW"u8); + archive.SetLength(8); + archive.Flush(); } Stream IArchiveFormatOperations.OpenEntry(Stream archive, string entryName, string? password) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(entryName); - var r = new MooseFsReader(archive); - var entry = r.Entries.FirstOrDefault(e => e.Name == entryName) + using var reader = new MooseFsReader(archive); + var entry = reader.Entries.FirstOrDefault(e => e.Name == entryName) ?? throw new FileNotFoundException($"MooseFS entry not found: {entryName}"); - var data = r.Extract(entry); + var data = reader.Extract(entry); return new BoundedEntryStream(new MemoryStream(data, writable: false), data.Length, leaveOpen: false); } } From a522d548984c324125cf81142b2f3ca8763098ed Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:05:28 +0200 Subject: [PATCH 066/165] Promote GlusterFS to real read-only brick view --- .../GlusterFsFormatDescriptor.cs | 97 +++++++++---------- 1 file changed, 47 insertions(+), 50 deletions(-) diff --git a/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs b/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs index f99f78848..f6b1487dd 100644 --- a/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs +++ b/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs @@ -6,31 +6,33 @@ namespace FileSystem.GlusterFs; /// -/// Stage 0 detection-only descriptor for GlusterFS. Honest fallback: -/// GlusterFS has no on-disk image format. A GlusterFS volume is a -/// logical aggregation of one or more "bricks", and every brick is -/// just a normal directory on a local POSIX filesystem (typically -/// XFS or ext4). Volume files live at their normal POSIX paths inside -/// the brick directory and carry GlusterFS state in extended -/// attributes (trusted.gfid, trusted.glusterfs.dht, -/// trusted.glusterfs.volume-id, trusted.glusterfs.pathinfo, -/// etc.). There is no superblock, no brick header, no portable -/// single-file representation that this image-based pipeline can -/// consume. +/// Read-only single-brick view for GlusterFS backing-store images. /// -/// We therefore stay Stage 0 permanently. The 0xCAFE5BAB magic -/// recognised here is a workbench-internal convention for hand-dumped -/// brick-object probes — it is not a real on-disk GlusterFS structure -/// and no real GlusterFS deployment will produce it. Promotion to R/O -/// would require walking a live directory tree and reading xattrs, -/// which is outside the image-stream contract enforced by -/// . +/// GlusterFS has no independent block format: a volume is a logical +/// collection of bricks and each brick is an export directory on an ordinary +/// filesystem with extended-attribute support. This descriptor therefore +/// delegates real backing images to the repository's XFS or ext2/3/4 reader and +/// surfaces the physical contents of one brick. It does not reconstruct the +/// distributed Gluster namespace. +/// +/// Detection is extension-only. XFS and ext images already belong to their +/// native descriptors and there is no Gluster-specific superblock magic with +/// which to distinguish a brick image automatically. The former workbench-only +/// 0xCAFE5BAB probe convention is deliberately not recognised as GlusterFS. +/// +/// Mutating maintenance operations are intentionally not advertised. A +/// Gluster brick relies on trusted.gfid and trusted.glusterfs.* +/// xattrs for identity, DHT layout, replication/heal and related state. The +/// current backing-filesystem rebuild paths do not preserve those attributes, +/// so create/modify/defrag/shrink/layout/wipe/purge would risk silent brick +/// corruption. /// /// References: /// -/// https://docs.gluster.org — official GlusterFS documentation (brick/xattr architecture) -/// https://github.com/gluster/glusterfs — canonical source -/// https://en.wikipedia.org/wiki/GlusterFS — Wikipedia overview +/// https://docs.gluster.org/en/latest/Administrator-Guide/Setting-Up-Volumes/ — a volume is a logical collection of export-directory bricks +/// https://docs.gluster.org/en/latest/Administrator-Guide/GlusterFS-Introduction/ — backing filesystems must support extended attributes +/// https://docs.gluster.org/en/latest/Administrator-Guide/Managing-Volumes/ — shrink/rebalance/fix-layout are cluster operations +/// https://github.com/gluster/glusterfs — canonical implementation, dual GPLv2/LGPLv3+ /// /// public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { @@ -42,7 +44,7 @@ public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveForma /// /// Gets the display name. /// - public string DisplayName => "GlusterFS"; + public string DisplayName => "GlusterFS brick"; /// /// Gets the category. /// @@ -51,7 +53,8 @@ public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveForma /// Gets the capabilities. /// public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest; + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | + FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; /// /// Gets the default extension. /// @@ -65,14 +68,9 @@ public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveForma /// public IReadOnlyList CompoundExtensions => []; /// - /// Gets the magic signatures. + /// Gets the magic signatures. Empty by design because GlusterFS has no separate on-disk magic. /// - public IReadOnlyList MagicSignatures => [ - // 0xCAFE5BAB — workbench-internal convention for hand-dumped brick - // object probes, NOT a real on-disk GlusterFS marker. GlusterFS - // itself has no on-disk header. See type-level doc. - new([0xCA, 0xFE, 0x5B, 0xAB], Offset: 0, Confidence: 0.90), - ]; + public IReadOnlyList MagicSignatures => []; /// /// Gets the methods. /// @@ -89,41 +87,40 @@ public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveForma /// Gets the description. /// public string Description => - "GlusterFS — Stage 0 (detection-only, permanent). A GlusterFS brick is a normal " + - "directory on a local POSIX filesystem (XFS/ext4); files live at their POSIX paths " + - "and metadata lives in xattrs (trusted.gfid, trusted.glusterfs.*). There is no " + - "on-disk image format, so no R/O promotion is possible from a single image stream. " + - "The 0xCAFE5BAB magic is a workbench-internal probe convention, not a real " + - "GlusterFS marker."; + "GlusterFS single-brick R/O backing-store view via XFS/ext delegation. GlusterFS has no " + + "standalone image format; the supplied .gluster image is treated as one brick's backing " + + "filesystem. The .glusterfs GFID index is hidden from normal listings. trusted.gfid and " + + "trusted.glusterfs.* xattrs remain preserved in the raw image but are not interpreted, so " + + "cluster namespace reconstruction and all mutations remain disabled."; /// - /// Lists the entries in the supplied container. + /// Lists the entries in the supplied brick backing-store image. /// public List List(Stream stream, string? password) { - var r = new GlusterFsReader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo( - i, e.Name, e.Size, e.Size, "Stored", e.IsDirectory, false, null)).ToList(); + using var reader = new GlusterFsReader(stream); + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo( + index, entry.Name, entry.Size, entry.Size, "Stored", entry.IsDirectory, false, null)).ToList(); } /// - /// Decodes the supplied input. + /// Extracts entries from the supplied single-brick physical view. /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new GlusterFsReader(stream); - foreach (var e in r.Entries) { - if (e.IsDirectory) continue; - if (files != null && !MatchesFilter(e.Name, files)) continue; - WriteFile(outputDir, e.Name, r.Extract(e)); + using var reader = new GlusterFsReader(stream); + foreach (var entry in reader.Entries) { + if (entry.IsDirectory) continue; + if (files != null && !MatchesFilter(entry.Name, files)) continue; + WriteFile(outputDir, entry.Name, reader.Extract(entry)); } } Stream IArchiveFormatOperations.OpenEntry(Stream archive, string entryName, string? password) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(entryName); - var r = new GlusterFsReader(archive); - var entry = r.Entries.FirstOrDefault(e => e.Name == entryName) - ?? throw new FileNotFoundException($"GlusterFS entry not found: {entryName}"); - var data = r.Extract(entry); + using var reader = new GlusterFsReader(archive); + var entry = reader.Entries.FirstOrDefault(candidate => candidate.Name == entryName) + ?? throw new FileNotFoundException($"GlusterFS brick entry not found: {entryName}"); + var data = reader.Extract(entry); return new BoundedEntryStream(new MemoryStream(data, writable: false), data.Length, leaveOpen: false); } } From 0f1e77a9504e341a83beee1844752044c7a4520a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 05:05:34 +0000 Subject: [PATCH 067/165] Update Lynx capability ledgers --- Hawkynt.FileFormats.Archives/README.md | 2 +- Hawkynt.FileFormats.FileSystems/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md index 9b682d0f7..7efd3edb4 100644 --- a/Hawkynt.FileFormats.Archives/README.md +++ b/Hawkynt.FileFormats.Archives/README.md @@ -127,7 +127,7 @@ Every State, Test, Maintenance, Compress/Decompress and Demux/Mux/Remux cell is | [LBR](https://en.wikipedia.org/wiki/LBR_(file_format)) | `Lbr` | `.lbr` | R/W | ✅ | defrag · wipe | | [gaby.de](http://www.gaby.de/cpm/manuals/archive/lbr.txt) | | LhF (LhFloppy) | `LhF` | `.lhf` | R/W | ✅ | defrag · wipe | Whole-disk archiver: entries are track_NNN.raw | [Aminet](https://aminet.net) | | [lrzip](https://en.wikipedia.org/wiki/Rzip#lrzip) | `Lrzip` | `.lrz` | WORM | ✅ | defrag | LZMA-wrapped subtype only; other lrzip subtypes are rejected; single data member | [GitHub](https://github.com/ckolivas/lrzip) | -| Lynx (Commodore) | `Lynx` | `.lnx` | R/W | ✅ | defrag · wipe | Stored entries only | [Archive Team](http://fileformats.archiveteam.org/wiki/Lynx_(Commodore_64)) | +| Lynx (Commodore) | `Lynx` | `.lnx` | R/W | ✅ | defrag · shrink · wipe | Stored entries only | [Archive Team](http://fileformats.archiveteam.org/wiki/Lynx_(Commodore_64)) | | [LHA / LZH](https://en.wikipedia.org/wiki/LHA_(file_format)) | `Lzh` | `.lzh` `.lha` | R/W | ✅ | defrag · wipe | | [GitHub](https://github.com/jca02266/lha) | | [LZX (Amiga)](https://en.wikipedia.org/wiki/LZX) | `LzxAmiga` | `.lzx` | R/W | ✅ | defrag · wipe | | [Aminet](https://aminet.net) | | [NuFX / ShrinkIt](https://en.wikipedia.org/wiki/ShrinkIt) | `NuFx` | `.shk` `.sdk` `.bxy` | R/W | ✅ | defrag · shrink · wipe | | [nulib.com](https://nulib.com/library/FTN.e08002.htm) | diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index d5725e500..20be5f36c 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -60,7 +60,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Apple IPSW](https://en.wikipedia.org/wiki/IPSW) | `Ipsw` | R/W | — | — | ✅ | — | — | ✅ | own reader + struct-parity tests | ZIP-based firmware archive | [IPSW](https://en.wikipedia.org/wiki/IPSW) | | [BIN/CUE](https://en.wikipedia.org/wiki/Cue_sheet_(computing)) | `BinCue` | R/W | — | — | ✅ | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [CUE sheet](https://wiki.hydrogenaud.io/index.php?title=Cue_sheet) | | [CDI](https://en.wikipedia.org/wiki/DiscJuggler) | `Cdi` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [DiscJuggler](https://en.wikipedia.org/wiki/DiscJuggler) | -| [Commodore Lynx archive](https://en.wikipedia.org/wiki/Commodore_64) | `Lynx` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 Lynx archive | [Lynx](https://ist.uwaterloo.ca/~schepers/formats/LNX.TXT) | +| [Commodore Lynx archive](https://en.wikipedia.org/wiki/Commodore_64) | `Lynx` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | C64 Lynx archive | [Lynx](https://ist.uwaterloo.ca/~schepers/formats/LNX.TXT) | | [DMG](https://en.wikipedia.org/wiki/Apple_Disk_Image) | `Dmg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | — | [Apple disk images](https://developer.apple.com/library/archive/documentation/DeveloperTools/Conceptual/SoftwareDistribution4/Concepts/sd_disk_images.html) | | [EnCase EWF (E01)](https://en.wikipedia.org/wiki/EnCase#Expert_Witness_File_Format) | `Ewf` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | EnCase E01 segment chain | [libewf documentation](https://github.com/libyal/libewf/tree/main/documentation) | | [Flattened Device Tree Blob](https://en.wikipedia.org/wiki/Devicetree) | `Dtb` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Flattened Device Tree walked as a pseudo-archive | [Devicetree specification](https://www.devicetree.org/specifications/) | From ed69b7116059644446fb445513ae4a82cb9aa331 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:05:40 +0200 Subject: [PATCH 068/165] Test transactional CSO ZSO edits and maintenance --- .../Cso/CsoInPlaceModifyTests.cs | 428 +++++++++--------- 1 file changed, 212 insertions(+), 216 deletions(-) diff --git a/Compression.Tests/Cso/CsoInPlaceModifyTests.cs b/Compression.Tests/Cso/CsoInPlaceModifyTests.cs index 38ffff14b..d9c16aa44 100644 --- a/Compression.Tests/Cso/CsoInPlaceModifyTests.cs +++ b/Compression.Tests/Cso/CsoInPlaceModifyTests.cs @@ -5,249 +5,245 @@ namespace Compression.Tests.Cso; -/// -/// Block-level in-place mutation for CSO v1 images. Two on-disk shapes are -/// exercised: (a) the new compressed block fits inside the old slot — header, -/// index, and adjacent block bytes remain byte-identical except for the -/// flag bit on the modified entry; (b) the new payload is larger than the -/// old slot — it lands at EOF and the index is patched to reflect the new -/// position while every block remains decodable through -/// . -/// [TestFixture] public class CsoInPlaceModifyTests { - private const int BlockSize = 2048; - /// - /// Produces an uncompressed payload of blocksCount × BlockSize bytes - /// where each block has a distinct fill so corruption is locally visible. - /// - private static byte[] BuildPayload(int blocksCount, byte seed) { - var data = new byte[blocksCount * BlockSize]; - for (var b = 0; b < blocksCount; ++b) - for (var i = 0; i < BlockSize; ++i) - data[b * BlockSize + i] = (byte)((seed + b * 7 + i) & 0xFF); - return data; + [TestCase(null, TestName = "WriteBlock_CsoV1")] + [TestCase("zso", TestName = "WriteBlock_Zso")] + [TestCase("cso2", TestName = "WriteBlock_CsoV2")] + [Category("RoundTrip")] + public void WriteBlock_RoundTripsModifiedAndUnmodifiedBlocks(string? variant) { + var payload = BuildPayload(3 * BlockSize, 7); + var image = CreateImage(payload, variant); + var replacement = new byte[BlockSize]; + new Random(1234).NextBytes(replacement); + + using var stream = Writable(image); + CsoInPlaceModifier.WriteBlock(stream, 1, replacement); + + var expected = payload.ToArray(); + replacement.CopyTo(expected, BlockSize); + Assert.That(ReadLogicalPayload(stream.ToArray()), Is.EqualTo(expected)); } - /// Builds a fresh CSO image around the supplied uncompressed payload. - private static byte[] BuildImage(byte[] payload) => CsoWriter.Build(payload); - - /// Reads the in-place index entries (without the flag bit) of a CSO stream. - private static (long[] Offsets, bool[] StoredFlags) ReadIndex(byte[] image) { - var blockSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(16, 4)); - var uncompressed = (long)BinaryPrimitives.ReadUInt64LittleEndian(image.AsSpan(8, 8)); - var blockCount = (int)((uncompressed + blockSize - 1) / blockSize); - var indexCount = blockCount + 1; - var offsets = new long[indexCount]; - var flags = new bool[indexCount]; - for (var i = 0; i < indexCount; ++i) { - var raw = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(24 + i * 4, 4)); - flags[i] = (raw & 0x8000_0000u) != 0; - offsets[i] = raw & 0x7FFF_FFFFu; - } - return (offsets, flags); - } - - [Test, Category("HappyPath")] - public void WriteBlock_FitsInPlace_LeavesHeaderAndOtherBlocksByteIdentical() { - var payload = BuildPayload(4, seed: 1); - var image = BuildImage(payload); - - // Choose a block to overwrite with HIGHLY-compressible content (zeros) - // so the new compressed bytes are guaranteed smaller than the old slot. - var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - var (oldOffsets, _) = ReadIndex(image); - var modifiedBlock = 1; - var oldSize = oldOffsets[modifiedBlock + 1] - oldOffsets[modifiedBlock]; - Assert.That(oldSize, Is.GreaterThan(0), "the chosen block must have non-zero on-disk size"); - - var zeros = new byte[BlockSize]; - CsoInPlaceModifier.WriteBlock(ms, modifiedBlock, zeros); - - var after = ms.ToArray(); - - // Stream length unchanged. - Assert.That(after.Length, Is.EqualTo(image.Length), - "in-place WriteBlock with a smaller payload must not grow the stream"); - - // Header bytes byte-identical (offsets 0..23). - Assert.That(after.AsSpan(0, 24).SequenceEqual(image.AsSpan(0, 24)), Is.True, - "header must be byte-identical after in-place WriteBlock"); - - // Index entries (offsets) unchanged for the in-place case; flag may flip. - var (newOffsets, _) = ReadIndex(after); - for (var i = 0; i < newOffsets.Length; ++i) - Assert.That(newOffsets[i], Is.EqualTo(oldOffsets[i]), - $"in-place modify must not move block {i} on disk"); - - // Every other block's bytes are byte-identical. - for (var i = 0; i < oldOffsets.Length - 1; ++i) { - if (i == modifiedBlock) continue; - var start = (int)oldOffsets[i]; - var end = (int)oldOffsets[i + 1]; - Assert.That(after.AsSpan(start, end - start).SequenceEqual(image.AsSpan(start, end - start)), - Is.True, $"block {i} bytes must not move when block {modifiedBlock} is rewritten in place"); - } + [Test, Category("Compatibility")] + public void WriteBlock_NonZeroIndexShift_CanonicalizesAndPreservesContent() { + var payload = BuildPayload(2 * BlockSize, 29); + var image = BuildShiftedStoredImage(payload); + var replacement = Enumerable.Repeat((byte)0xA5, BlockSize).ToArray(); + + using var stream = Writable(image); + CsoInPlaceModifier.WriteBlock(stream, 0, replacement); + var after = stream.ToArray(); + + Assert.That(after[21], Is.Zero, "transactional repack should canonicalize index_shift to zero"); + var expected = payload.ToArray(); + replacement.CopyTo(expected, 0); + Assert.That(ReadLogicalPayload(after), Is.EqualTo(expected)); } - [Test, Category("HappyPath")] - public void WriteBlock_FitsInPlace_RoundTripsModifiedAndUnmodifiedBlocks() { - var payload = BuildPayload(3, seed: 42); - var image = BuildImage(payload); - - var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - - var zeros = new byte[BlockSize]; - CsoInPlaceModifier.WriteBlock(ms, 1, zeros); - - // Extract each block and assert the modified one is zeros, others are intact. - var desc = new CsoFormatDescriptor(); - var tmp = Path.Combine(Path.GetTempPath(), "cso_inplace_test_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(tmp); - try { - ms.Position = 0; - desc.Extract(ms, tmp, null, ["block_00000.bin", "block_00001.bin", "block_00002.bin"]); - - var b0 = File.ReadAllBytes(Path.Combine(tmp, "blocks/block_00000.bin")); - var b1 = File.ReadAllBytes(Path.Combine(tmp, "blocks/block_00001.bin")); - var b2 = File.ReadAllBytes(Path.Combine(tmp, "blocks/block_00002.bin")); - - // List reports each block's compressed size; descriptor's Extract writes - // the on-disk (still-compressed) bytes. We only assert structural - // properties — the modified entry must round-trip a decompressed-zero - // slab when inflated, which the FULL.cso path covers. - Assert.That(b0.Length, Is.GreaterThan(0)); - Assert.That(b1.Length, Is.GreaterThan(0), "modified block on disk has compressed-zeros payload"); - Assert.That(b2.Length, Is.GreaterThan(0)); - } finally { - Directory.Delete(tmp, recursive: true); - } + [Test, Category("RoundTrip")] + public void Descriptor_Add_ReplacesSeveralBlocksInOneOperation() { + var payload = BuildPayload(4 * BlockSize, 3); + var image = CreateImage(payload, "zso"); + var block0 = Enumerable.Repeat((byte)0x11, BlockSize).ToArray(); + var block3 = Enumerable.Repeat((byte)0xCC, BlockSize).ToArray(); + + using var stream = Writable(image); + IArchiveModifiable modifier = new CsoFormatDescriptor(); + modifier.Add(stream, [ + ArchiveInputInfo.InMemory("blocks/block_00000.bin", block0), + ArchiveInputInfo.InMemory("blocks/block_00003.bin", block3), + ]); + + var expected = payload.ToArray(); + block0.CopyTo(expected, 0); + block3.CopyTo(expected, 3 * BlockSize); + Assert.That(ReadLogicalPayload(stream.ToArray()), Is.EqualTo(expected)); } - [Test, Category("HappyPath")] - public void WriteBlock_LargerThanSlot_HeaderPrefixUnchanged_OtherBlocksDecodable() { - // Build payload where blocks are highly compressible so the writer emits - // small compressed bytes, then over-write block 0 with random data whose - // DEFLATE output is larger than the original tiny slot — forces the - // append-at-EOF branch. - var payload = new byte[3 * BlockSize]; // zeros → DEFLATE compresses tiny - var image = BuildImage(payload); - var (oldOffsets, _) = ReadIndex(image); - var oldBlock0Size = oldOffsets[1] - oldOffsets[0]; + [Test, Category("ErrorHandling")] + public void WriteBlock_BlockIndexOutOfRange_Throws() { + using var stream = Writable(CreateImage(BuildPayload(2 * BlockSize, 9))); + Assert.That( + () => CsoInPlaceModifier.WriteBlock(stream, 99, new byte[BlockSize]), + Throws.InstanceOf()); + } - // Random bytes don't compress; their DEFLATE output will be > oldBlock0Size. - var random = new byte[BlockSize]; - new Random(1).NextBytes(random); + [Test, Category("ErrorHandling")] + public void WriteBlock_WrongPayloadSize_Throws() { + using var stream = Writable(CreateImage(BuildPayload(2 * BlockSize, 9))); + Assert.That( + () => CsoInPlaceModifier.WriteBlock(stream, 0, new byte[BlockSize - 1]), + Throws.InstanceOf()); + } - var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - var oldLen = (int)ms.Length; - CsoInPlaceModifier.WriteBlock(ms, 0, random); + [Test, Category("Capabilities")] + public void Descriptor_AdvertisesReadWriteAndMaintenanceInterfaces() { + var descriptor = new CsoFormatDescriptor(); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + } - var after = ms.ToArray(); + [Test, Category("Maintenance")] + public void Defragment_DropsUnindexedTailAndPreservesLogicalIso() { + var payload = BuildPayload(3 * BlockSize, 41); + var image = CreateImage(payload, "cso2"); + using var stream = Writable(AppendGarbage(image, 193)); + var oldLength = stream.Length; - // Stream grew because the new payload didn't fit in place. - Assert.That(after.Length, Is.GreaterThan(oldLen), - "grow-then-append branch must extend the stream"); + ((IArchiveDefragmentable)new CsoFormatDescriptor()).Defragment(stream); - // Header pre-fix unchanged (offsets 0..23 carry magic + uncompressed_size + - // block_size + version + align; they are NOT touched by in-place writes). - Assert.That(after.AsSpan(0, 24).SequenceEqual(image.AsSpan(0, 24)), Is.True, - "the 24-byte header must remain byte-identical after a grow-append modify"); + Assert.That(stream.Length, Is.LessThan(oldLength)); + Assert.That(ReadLogicalPayload(stream.ToArray()), Is.EqualTo(payload)); + Assert.That(stream.ToArray()[21], Is.Zero); + } - // Index moved: block 0 now points past the old tail. - var (newOffsets, _) = ReadIndex(after); - Assert.That(newOffsets[0], Is.GreaterThanOrEqualTo(oldLen), - "moved block 0 must land past the original EOF"); + [Test, Category("Maintenance")] + public void WipeUnusedSpace_ZeroesCanonicalizedTailWithoutChangingLength() { + var payload = BuildPayload(2 * BlockSize, 55); + var core = CreateImage(payload, "zso"); + using var stream = Writable(AppendGarbage(core, 257)); + var originalLength = stream.Length; - // Every block still decodable via FULL.cso round-trip (header + index + bodies - // describe a coherent CSO container). - var desc = new CsoFormatDescriptor(); - ms.Position = 0; - var entries = desc.List(ms, null); - Assert.That(entries.Count(e => e.Name.StartsWith("blocks/block_")), Is.EqualTo(3), - "post-grow image must still surface 3 synthetic block entries"); + var wiped = ((IWipeEmpty)new CsoFormatDescriptor()).WipeUnusedSpace(stream); + + Assert.That(wiped, Is.EqualTo(257)); + Assert.That(stream.Length, Is.EqualTo(originalLength)); + Assert.That(stream.ToArray().AsSpan(core.Length).ToArray(), Is.All.Zero); + Assert.That(ReadLogicalPayload(stream.ToArray()), Is.EqualTo(payload)); } - [Test, Category("ErrorHandling")] - public void WriteBlock_BlockIndexOutOfRange_Throws() { - var image = BuildImage(BuildPayload(2, seed: 9)); - using var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - Assert.That( - () => CsoInPlaceModifier.WriteBlock(ms, 99, new byte[BlockSize]), - Throws.InstanceOf(), - "out-of-range block index must throw, not corrupt the image"); + [Test, Category("Maintenance")] + public void Shrink_DropsUnindexedTailAndNeverGrows() { + var payload = BuildPayload(2 * BlockSize, 71); + var sourceBytes = AppendGarbage(CreateImage(payload), 311); + using var source = new MemoryStream(sourceBytes, writable: false); + using var target = new MemoryStream(); + + ((IArchiveShrinkable)new CsoFormatDescriptor()).Shrink(source, target); + + Assert.That(target.Length, Is.LessThan(sourceBytes.Length)); + Assert.That(ReadLogicalPayload(target.ToArray()), Is.EqualTo(payload)); } - [Test, Category("ErrorHandling")] - public void WriteBlock_WrongPayloadSize_Throws() { - var image = BuildImage(BuildPayload(2, seed: 9)); - using var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - Assert.That( - () => CsoInPlaceModifier.WriteBlock(ms, 0, new byte[BlockSize - 1]), - Throws.InstanceOf(), - "payload must be exactly block_size bytes"); + [Test, Category("Maintenance")] + public void RebuildStreaming_ReblocksWithoutChangingLogicalIso() { + var payload = BuildPayload(5 * BlockSize, 87); + var sourceBytes = CreateImage(payload, "cso2"); + using var source = new MemoryStream(sourceBytes, writable: false); + using var target = new MemoryStream(); + + ((ILayoutOptimizable)new CsoFormatDescriptor()).RebuildStreaming( + source, target, new LayoutRebuildOptions { UnitSize = 4096 }); + + var rebuilt = target.ToArray(); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(rebuilt.AsSpan(16, 4)), Is.EqualTo(4096)); + Assert.That(ReadLogicalPayload(rebuilt), Is.EqualTo(payload)); } - [Test, Category("RoundTrip")] - public void MutateThenExtract_RoundTrip() { - var payload = BuildPayload(2, seed: 7); - var image = BuildImage(payload); - - using var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - - // Mutate block 0 to all-zeros via the descriptor's IArchiveModifiable surface. - IArchiveModifiable mod = new CsoFormatDescriptor(); - mod.Add(ms, [ArchiveInputInfo.InMemory("blocks/block_00000.bin", new byte[BlockSize])]); - - // Extract and verify FULL.cso re-reads the mutated image without error. - var desc = new CsoFormatDescriptor(); - var tmp = Path.Combine(Path.GetTempPath(), "cso_mutate_extract_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(tmp); - try { - ms.Position = 0; - desc.Extract(ms, tmp, null, ["FULL.cso", "metadata.ini"]); - Assert.That(File.Exists(Path.Combine(tmp, "FULL.cso")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "metadata.ini")), Is.True); - } finally { - Directory.Delete(tmp, recursive: true); + [Test, Category("Maintenance")] + public void LayoutMap_DeclaresOnlyTrailingUnindexedBytesFree() { + var image = AppendGarbage(CreateImage(BuildPayload(BlockSize, 101)), 64); + using var stream = new MemoryStream(image, writable: false); + + var layout = ((IArchiveLayoutMap)new CsoFormatDescriptor()).EnumerateLayout(stream).ToArray(); + + Assert.That(layout[0].Kind, Is.EqualTo(DefragBlockKind.MetadataReserved)); + Assert.That(layout.Any(e => e.Kind == DefragBlockKind.Used), Is.True); + var free = layout.Single(e => e.Kind == DefragBlockKind.Free); + Assert.That(free.Length, Is.EqualTo(64)); + Assert.That(free.Offset + free.Length, Is.EqualTo(image.Length)); + } + + [Test, Category("Maintenance")] + public void Purge_LeavesValidEmptyContainerOfSameVariant() { + using var stream = Writable(CreateImage(BuildPayload(2 * BlockSize, 111), "zso")); + + ((IArchivePurgeable)new CsoFormatDescriptor()).Purge(stream); + + var bytes = stream.ToArray(); + Assert.That(System.Text.Encoding.ASCII.GetString(bytes, 0, 4), Is.EqualTo("ZISO")); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(bytes.AsSpan(8, 8)), Is.Zero); + stream.Position = 0; + var entries = new CsoFormatDescriptor().List(stream, null); + Assert.That(entries.Count(e => e.Name.StartsWith("blocks/block_", StringComparison.Ordinal)), Is.Zero); + } + + private static MemoryStream Writable(byte[] bytes) { + var stream = new MemoryStream(); + stream.Write(bytes); + stream.Position = 0; + return stream; + } + + private static byte[] CreateImage(byte[] payload, string? variant = null) { + var creator = (IArchiveCreatable)new CsoFormatDescriptor(); + using var stream = new MemoryStream(); + creator.Create(stream, [ArchiveInputInfo.InMemory("disc.iso", payload)], new FormatCreateOptions(variant)); + return stream.ToArray(); + } + + private static byte[] ReadLogicalPayload(byte[] image) { + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image, writable: false); + var names = descriptor.List(stream, null) + .Where(e => !e.IsDirectory && e.Name.StartsWith("blocks/block_", StringComparison.Ordinal)) + .OrderBy(e => e.Index) + .Select(e => e.Name) + .ToArray(); + using var result = new MemoryStream(); + foreach (var name in names) { + stream.Position = 0; + result.Write(descriptor.ExtractEntryToMemory(stream, name, null)); } + return result.ToArray(); } - [Test, Category("HappyPath")] - public void Descriptor_CapabilitiesAdvertiseCreateAndModify() { - var d = new CsoFormatDescriptor(); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True, - "Cso descriptor must advertise CanCreate"); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True, - "Cso descriptor must advertise CanModify"); - Assert.That(d, Is.InstanceOf()); - Assert.That(d, Is.InstanceOf()); + private static byte[] AppendGarbage(byte[] image, int count) { + var result = new byte[image.Length + count]; + image.CopyTo(result, 0); + result.AsSpan(image.Length).Fill(0xD7); + return result; } - [Test, Category("RoundTrip")] - public void Create_FromInputs_ProducesValidCso() { - // Build a 4 KB uncompressed payload by feeding two 2 KB inputs in order. - var first = new byte[BlockSize]; Array.Fill(first, (byte)0xAA); - var second = new byte[BlockSize]; Array.Fill(second, (byte)0x55); - - var d = (IArchiveCreatable)new CsoFormatDescriptor(); - using var ms = new MemoryStream(); - d.Create(ms, - [ArchiveInputInfo.InMemory("part1.bin", first), - ArchiveInputInfo.InMemory("part2.bin", second)], - new FormatCreateOptions()); - - ms.Position = 0; - var entries = new CsoFormatDescriptor().List(ms, null); - Assert.That(entries.Count(e => e.Name.StartsWith("blocks/block_")), Is.EqualTo(2)); - Assert.That(entries.First(e => e.Name == "FULL.cso").OriginalSize, Is.EqualTo(ms.Length)); + private static byte[] BuildShiftedStoredImage(byte[] payload) { + Assert.That(payload.Length, Is.EqualTo(2 * BlockSize)); + const byte shift = 1; + const int indexEnd = 24 + 3 * 4; + const int dataStart = 40; // even and deliberately padded beyond the index table. + var result = new byte[dataStart + payload.Length]; + "CISO"u8.CopyTo(result); + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(4, 4), 24); + BinaryPrimitives.WriteUInt64LittleEndian(result.AsSpan(8, 8), checked((ulong)payload.Length)); + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(16, 4), BlockSize); + result[20] = 1; + result[21] = shift; + + var first = checked((uint)(dataStart >> shift)) | 0x8000_0000u; + var second = checked((uint)((dataStart + BlockSize) >> shift)) | 0x8000_0000u; + var end = checked((uint)((dataStart + payload.Length) >> shift)); + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(24, 4), first); + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(28, 4), second); + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(32, 4), end); + Assert.That(indexEnd, Is.EqualTo(36)); + payload.CopyTo(result, dataStart); + return result; + } + + private static byte[] BuildPayload(int length, byte seed) { + var data = new byte[length]; + for (var i = 0; i < data.Length; ++i) + data[i] = (byte)(seed + i * 13 + i / 97); + return data; } } From b1ae17624478ccbcefacf4b476a57ef48333bac9 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:05:53 +0200 Subject: [PATCH 069/165] Test real GlusterFS brick backing-store reads --- .../GlusterFs/GlusterFsDetectionTests.cs | 158 +++++++++++------- 1 file changed, 101 insertions(+), 57 deletions(-) diff --git a/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs b/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs index 02a16ffbc..f670dae2a 100644 --- a/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs +++ b/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs @@ -7,78 +7,122 @@ namespace Compression.Tests.GlusterFs; [TestFixture] public class GlusterFsDetectionTests { - private static byte[] BuildMinimal(int payloadLen = 128) { - var image = new byte[8 + payloadLen]; - image[0] = 0xCA; image[1] = 0xFE; image[2] = 0x5B; image[3] = 0xAB; - for (var i = 0; i < payloadLen; i++) image[8 + i] = (byte)(i & 0xFF); - return image; + private static byte[] BuildXfsBrick() { + var writer = new FileSystem.Xfs.XfsWriter(); + writer.AddFile("data/hello.txt", "hello from xfs brick"u8.ToArray()); + writer.AddFile(".glusterfs/ab/internal-gfid", "backend index"u8.ToArray()); + using var stream = new MemoryStream(); + writer.WriteTo(stream); + return stream.ToArray(); } - [Test, Category("HappyPath")] - public void Detector_IdentifiesByMagic() { - var d = new GlusterFsFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("GlusterFs")); - Assert.That(d.Extensions, Does.Contain(".gluster")); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(1)); - Assert.That(d.MagicSignatures[0].Bytes, Is.EqualTo(new byte[] { 0xCA, 0xFE, 0x5B, 0xAB })); - Assert.That(d, Is.Not.InstanceOf()); + private static byte[] BuildExtBrick() { + var writer = new FileSystem.Ext.ExtWriter(); + writer.AddFile("data/hello.txt", "hello from ext brick"u8.ToArray()); + writer.AddFile(".glusterfs/cd/internal-gfid", "backend index"u8.ToArray()); + return writer.Build(); } [Test, Category("HappyPath")] - public void List_ReturnsTwoEntries() { - var d = new GlusterFsFormatDescriptor(); - using var ms = new MemoryStream(BuildMinimal(payloadLen: 256)); - var entries = d.List(ms, password: null); - var names = entries.Select(e => e.Name).ToList(); - Assert.That(names, Is.EquivalentTo(new[] { "metadata.ini", "gluster-brick.bin" })); + public void Descriptor_UsesExtensionOnlyDetectionAndAdvertisesReadOnlyBrickView() { + var descriptor = new GlusterFsFormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor.Id, Is.EqualTo("GlusterFs")); + Assert.That(descriptor.Extensions, Does.Contain(".gluster")); + Assert.That(descriptor.MagicSignatures, Is.Empty, + "GlusterFS has no standalone block-format magic and must not collide with XFS/ext detection."); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanList), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanExtract), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.SupportsDirectories), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + }); } - [Test, Category("Stub")] - public void Description_FlagsStage0Permanent() { - var d = new GlusterFsFormatDescriptor(); - var lower = d.Description.ToLowerInvariant(); - Assert.That(lower, Does.Contain("detection-only")); - Assert.That(lower, Does.Contain("stage 0")); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + [Test, Category("HappyPath")] + public void XfsBackingStore_ListsBrickFilesAndHidesGlusterInternalIndex() { + var descriptor = new GlusterFsFormatDescriptor(); + using var stream = new MemoryStream(BuildXfsBrick()); + + var names = descriptor.List(stream, password: null).Select(entry => entry.Name).ToArray(); + + Assert.Multiple(() => { + Assert.That(names, Does.Contain("metadata.ini")); + Assert.That(names, Does.Contain("brick/data/hello.txt")); + Assert.That(names.Any(name => name.Contains(".glusterfs", StringComparison.Ordinal)), Is.False); + }); } - [Test, Category("Stub")] - public void Description_DocumentsHonestFallback() { - // The honest-fallback rule: a GlusterFS brick is a normal directory on a - // local POSIX FS and state lives in xattrs — no on-disk image, no R/O - // promotion possible from a single image stream. - var d = new GlusterFsFormatDescriptor(); - var lower = d.Description.ToLowerInvariant(); - Assert.That(lower, Does.Contain("brick")); - Assert.That(lower, Does.Contain("director")); // "directory" or "directories" - Assert.That(lower, Does.Contain("xattr")); - Assert.That(lower, Does.Contain("no on-disk image")); - Assert.That(lower, Does.Contain("probe")); // magic is workbench probe, not real + [Test, Category("HappyPath")] + public void ExtBackingStore_ListsAndExtractsBrickFile() { + using var stream = new MemoryStream(BuildExtBrick()); + using var reader = new GlusterFsReader(stream); + + var entry = reader.Entries.Single(candidate => candidate.Name == "brick/data/hello.txt"); + + Assert.Multiple(() => { + Assert.That(reader.ValidHeader, Is.True); + Assert.That(reader.BackingFileSystem, Is.EqualTo("ext")); + Assert.That(Encoding.UTF8.GetString(reader.Extract(entry)), Is.EqualTo("hello from ext brick")); + Assert.That(reader.Entries.Any(candidate => candidate.Name.Contains(".glusterfs", StringComparison.Ordinal)), Is.False); + }); } - [Test, Category("Stub")] - public void Metadata_DeclaresStage0PermanentAndExplainsWhyROBlocked() { - // The synthetic metadata.ini must spell out (a) stage=0, (b) stage is - // permanent, (c) the concrete reason R/O promotion is blocked. This is the - // contract the parent-task pipeline checks against when triaging FSes. - var d = new GlusterFsFormatDescriptor(); - using var ms = new MemoryStream(BuildMinimal(payloadLen: 64)); - var workDir = Path.Combine(Path.GetTempPath(), $"GlusterFsMeta_{Guid.NewGuid():N}"); + [Test, Category("HappyPath")] + public void XfsBackingStore_ExtractsThroughDescriptor() { + var descriptor = new GlusterFsFormatDescriptor(); + using var stream = new MemoryStream(BuildXfsBrick()); + var workDir = Path.Combine(Path.GetTempPath(), $"GlusterFsBrick_{Guid.NewGuid():N}"); Directory.CreateDirectory(workDir); try { - d.Extract(ms, workDir, password: null, files: ["metadata.ini"]); - var path = Path.Combine(workDir, "metadata.ini"); - Assert.That(File.Exists(path), Is.True); - var text = File.ReadAllText(path, Encoding.UTF8); - Assert.That(text, Does.Contain("stage=0")); - Assert.That(text, Does.Contain("stage_permanent=true")); - Assert.That(text, Does.Contain("ro_blocked_reason=")); - Assert.That(text, Does.Contain("xattr")); - Assert.That(text, Does.Contain("no on-disk image format")); - Assert.That(text, Does.Contain("magic_kind=workbench-internal probe")); + descriptor.Extract(stream, workDir, password: null, files: ["brick/data/hello.txt"]); + var path = Path.Combine(workDir, "brick", "data", "hello.txt"); + Assert.That(File.ReadAllText(path, Encoding.UTF8), Is.EqualTo("hello from xfs brick")); } finally { try { Directory.Delete(workDir, recursive: true); } catch { /* best effort */ } } } + + [Test, Category("Regression")] + public void WorkbenchProbeMagic_IsNoLongerAcceptedAsGlusterFs() { + byte[] oldSyntheticProbe = [0xCA, 0xFE, 0x5B, 0xAB, 0, 0, 0, 0]; + using var stream = new MemoryStream(oldSyntheticProbe); + + var error = Assert.Throws(() => new GlusterFsReader(stream)); + + Assert.That(error!.Message, Does.Contain("backing-store image")); + } + + [Test, Category("Regression")] + public void Descriptor_DoesNotAdvertiseMutationsThatWouldDiscardGlusterXattrs() { + var descriptor = new GlusterFsFormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + }); + } + + [Test, Category("HappyPath")] + public void Metadata_DescribesPhysicalViewAndMutationBlocker() { + using var stream = new MemoryStream(BuildXfsBrick()); + using var reader = new GlusterFsReader(stream); + var metadata = reader.Entries.Single(entry => entry.Name == "metadata.ini"); + var text = Encoding.UTF8.GetString(reader.Extract(metadata)); + + Assert.Multiple(() => { + Assert.That(text, Does.Contain("parse_status=single-brick-read-only")); + Assert.That(text, Does.Contain("backing_fs=xfs")); + Assert.That(text, Does.Contain("view=physical single-brick namespace")); + Assert.That(text, Does.Contain("xattrs=preserved in image but not interpreted")); + Assert.That(text, Does.Contain("cluster_namespace_reconstruction=false")); + Assert.That(text, Does.Contain("trusted.gfid/trusted.glusterfs.*")); + }); + } } From 8f1d948ba65e3024dfd54564fe2960c63650b095 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:05:58 +0200 Subject: [PATCH 070/165] Avoid capturing span in NRG test lambda --- Compression.Tests/Nrg/NrgTests.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Compression.Tests/Nrg/NrgTests.cs b/Compression.Tests/Nrg/NrgTests.cs index 8f079d047..a3a47fe3e 100644 --- a/Compression.Tests/Nrg/NrgTests.cs +++ b/Compression.Tests/Nrg/NrgTests.cs @@ -80,11 +80,9 @@ public void Create_WritesRealV2ChunkChain() { var position = trailerOffset; var etn2 = ReadChunk(image, ref position, "ETN2", 32); - Assert.Multiple(() => { - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(etn2), Is.Zero); - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(etn2[8..]), Is.EqualTo((ulong)trailerOffset)); - Assert.That(etn2[19], Is.Zero, "cooked Mode 1"); - }); + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(etn2), Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(etn2[8..]), Is.EqualTo((ulong)trailerOffset)); + Assert.That(etn2[19], Is.Zero, "cooked Mode 1"); var sinf = ReadChunk(image, ref position, "SINF", 4); Assert.That(BinaryPrimitives.ReadUInt32BigEndian(sinf), Is.EqualTo(1)); @@ -109,7 +107,6 @@ private static ReadOnlySpan ReadChunk(byte[] image, ref int position, stri public void Read_UsesEtn2TrackOffset_InsteadOfAssumingOffsetZero() { var payload = "track-offset"u8.ToArray(); var image = BuildOffsetTrackNrg("OFFSET.BIN", payload); - Assert.That(ReadFile(image, "OFFSET.BIN"), Is.EqualTo(payload)); } From 776a50c41b1a645e98c59a4cef345aecbc715ec1 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:05:58 +0200 Subject: [PATCH 071/165] fix(onefs): preserve reader constructor compatibility --- FileSystems/FileSystem.OneFs/OneFsReader.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/FileSystems/FileSystem.OneFs/OneFsReader.cs b/FileSystems/FileSystem.OneFs/OneFsReader.cs index ea6c6d8e0..303ad4e7e 100644 --- a/FileSystems/FileSystem.OneFs/OneFsReader.cs +++ b/FileSystems/FileSystem.OneFs/OneFsReader.cs @@ -83,12 +83,19 @@ public sealed class OneFsReader : IDisposable { /// Gets the byte length of the opaque source image. public long ImageSize { get; } + /// + /// Initializes a conservative OneFS image reader without consuming or owning + /// the source stream. Kept as the original one-argument public constructor for + /// binary/source API compatibility. + /// + public OneFsReader(Stream stream) : this(stream, leaveOpen: true) { } + /// /// Initializes a conservative OneFS image reader without consuming the image. /// /// Readable, seekable raw image stream. /// Whether disposing the reader leaves the source open. - public OneFsReader(Stream stream, bool leaveOpen = true) { + public OneFsReader(Stream stream, bool leaveOpen) { ArgumentNullException.ThrowIfNull(stream); if (!stream.CanRead) throw new ArgumentException("OneFS inspection requires a readable stream.", nameof(stream)); From 5cb80f3d310d6b2d079768531abbb98b15910da2 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:06:01 +0200 Subject: [PATCH 072/165] Remove temporary Lynx ledger helper --- .github/workflows/patch-lynx-ledgers.yml | 50 ------------------------ 1 file changed, 50 deletions(-) delete mode 100644 .github/workflows/patch-lynx-ledgers.yml diff --git a/.github/workflows/patch-lynx-ledgers.yml b/.github/workflows/patch-lynx-ledgers.yml deleted file mode 100644 index f804819e3..000000000 --- a/.github/workflows/patch-lynx-ledgers.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Patch Lynx ledgers - -on: - push: - branches: - - feature/commodore-lynx-maintenance - -permissions: - contents: write - -jobs: - patch: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: feature/commodore-lynx-maintenance - - - name: Update generated capability rows - shell: python - run: | - from pathlib import Path - - replacements = { - Path("Hawkynt.FileFormats.Archives/README.md"): ( - "| Lynx (Commodore) | `Lynx` | `.lnx` | R/W | ✅ | defrag · wipe | Stored entries only | [Archive Team](http://fileformats.archiveteam.org/wiki/Lynx_(Commodore_64)) |", - "| Lynx (Commodore) | `Lynx` | `.lnx` | R/W | ✅ | defrag · shrink · wipe | Stored entries only | [Archive Team](http://fileformats.archiveteam.org/wiki/Lynx_(Commodore_64)) |", - ), - Path("Hawkynt.FileFormats.FileSystems/README.md"): ( - "| [Commodore Lynx archive](https://en.wikipedia.org/wiki/Commodore_64) | `Lynx` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 Lynx archive | [Lynx](https://ist.uwaterloo.ca/~schepers/formats/LNX.TXT) |", - "| [Commodore Lynx archive](https://en.wikipedia.org/wiki/Commodore_64) | `Lynx` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | C64 Lynx archive | [Lynx](https://ist.uwaterloo.ca/~schepers/formats/LNX.TXT) |", - ), - } - - for path, (old, new) in replacements.items(): - text = path.read_text(encoding="utf-8") - if text.count(old) != 1: - raise SystemExit(f"Expected exactly one current Lynx row in {path}") - path.write_text(text.replace(old, new), encoding="utf-8", newline="\n") - - - name: Commit capability ledgers - shell: bash - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add Hawkynt.FileFormats.Archives/README.md Hawkynt.FileFormats.FileSystems/README.md - git diff --cached --quiet && exit 0 - git commit -m "Update Lynx capability ledgers" - git push origin HEAD:feature/commodore-lynx-maintenance From 9fdc9d35bf108fac9c1cab45a4a1af9ed6edf97a Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:06:03 +0200 Subject: [PATCH 073/165] + conservative ldiskfs maintenance for Lustre targets --- .../LustreLdiskfsMaintenance.cs | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs diff --git a/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs b/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs new file mode 100644 index 000000000..27bfb5410 --- /dev/null +++ b/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs @@ -0,0 +1,290 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using Compression.Registry; +using FileSystem.Ext; + +namespace FileSystem.Lustre; + +/// +/// Conservative maintenance primitives for an offline Lustre ldiskfs MDT/OST backing image. +/// They deliberately operate below the Lustre namespace layer: no inode, directory or Lustre +/// xattr is reconstructed. Only allocation-bitmap-proven free blocks are wiped, and shrinking +/// uses the ext in-place geometry editor so every surviving allocated block remains intact. +/// +/// +/// The layout facts used here come from the Linux ext4 on-disk documentation. In particular, +/// block-group descriptors locate the block bitmaps, one clear bitmap bit means one free block, +/// and a BLOCK_UNINIT group does not carry a trustworthy on-disk bitmap. Lustre documents +/// ldiskfs as its ext4-derived backing filesystem. No external implementation code is copied. +/// +internal static class LustreLdiskfsMaintenance { + private const int SuperblockOffset = 1024; + private const ushort ExtMagic = 0xEF53; + + private const uint CompatSparseSuper2 = 0x0200; + + private const uint IncompatRecover = 0x0004; + private const uint IncompatJournalDev = 0x0008; + private const uint IncompatMetaBg = 0x0010; + private const uint Incompat64Bit = 0x0080; + + private const uint RoCompatBigalloc = 0x0200; + private const uint RoCompatReadOnly = 0x1000; + + private const ushort BgBlockUninit = 0x0002; + + private readonly record struct Geometry( + int BlockSize, + ulong TotalBlocks, + uint FirstDataBlock, + uint BlocksPerGroup, + uint GroupCount, + int DescriptorSize, + long DescriptorTableOffset, + uint FeatureCompat, + uint FeatureIncompat, + uint FeatureRoCompat); + + /// + /// Wipes only complete blocks that the initialized ldiskfs allocation bitmap proves free. + /// BLOCK_UNINIT groups are skipped rather than inferred, because ext4 explicitly permits + /// their on-disk bitmaps to contain meaningless data while fixed metadata is still present. + /// + public static long WipeFreeBlocks(Stream image) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("Lustre wipe requires a readable, writable, seekable stream.", nameof(image)); + + var geometry = ReadGeometry(image); + ValidateWritableProfile(geometry); + + var descriptor = new byte[geometry.DescriptorSize]; + var bitmap = new byte[geometry.BlockSize]; + var zeros = new byte[Math.Min(64 * 1024, Math.Max(geometry.BlockSize, 4096))]; + long wiped = 0; + + for (uint group = 0; group < geometry.GroupCount; ++group) { + ReadDescriptor(image, geometry, group, descriptor); + var flags = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.AsSpan(0x12, 2)); + if ((flags & BgBlockUninit) != 0) + continue; + + var bitmapBlock = ReadBlockBitmapBlock(descriptor, geometry.DescriptorSize); + if (bitmapBlock >= geometry.TotalBlocks) + throw new InvalidDataException($"Lustre/ldiskfs: group {group} block bitmap points outside the volume ({bitmapBlock})."); + + var bitmapOffset = CheckedByteOffset(bitmapBlock, geometry.BlockSize); + if (bitmapOffset + geometry.BlockSize > image.Length) + throw new InvalidDataException($"Lustre/ldiskfs: group {group} block bitmap is truncated."); + + image.Position = bitmapOffset; + image.ReadExactly(bitmap); + + var groupFirst = (ulong)geometry.FirstDataBlock + (ulong)group * geometry.BlocksPerGroup; + if (groupFirst >= geometry.TotalBlocks) + break; + var blocksInGroup = (int)Math.Min((ulong)geometry.BlocksPerGroup, geometry.TotalBlocks - groupFirst); + + var runStart = -1; + for (var bit = 0; bit <= blocksInGroup; ++bit) { + var isFree = bit < blocksInGroup && (bitmap[bit >> 3] & (1 << (bit & 7))) == 0; + if (isFree) { + if (runStart < 0) runStart = bit; + continue; + } + + if (runStart < 0) + continue; + + var runBlocks = bit - runStart; + var firstBlock = groupFirst + (uint)runStart; + var offset = CheckedByteOffset(firstBlock, geometry.BlockSize); + var length = checked((long)runBlocks * geometry.BlockSize); + if (offset + length > image.Length) + throw new InvalidDataException($"Lustre/ldiskfs: free run in group {group} extends beyond the image."); + + ZeroRange(image, offset, length, zeros); + wiped = checked(wiped + length); + runStart = -1; + } + } + + image.Flush(); + return wiped; + } + + /// + /// Copies to and trims trailing + /// bitmap-free ldiskfs blocks using the existing ext in-place shrinker. There is no + /// extract/re-create fallback: such a fallback would lose Lustre-specific xattrs and + /// target metadata even when ordinary file contents happened to round-trip. + /// + public static void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("Lustre shrink requires a readable, seekable input stream.", nameof(input)); + if (!output.CanRead || !output.CanWrite || !output.CanSeek) + throw new ArgumentException("Lustre shrink requires a readable, writable, seekable output stream.", nameof(output)); + if (ReferenceEquals(input, output)) + throw new ArgumentException("Lustre shrink requires a distinct output stream so failure cannot partially rewrite the source.", nameof(output)); + + var geometry = ReadGeometry(input); + ValidateWritableProfile(geometry); + ValidateShrinkProfile(input, geometry); + + input.Position = 0; + output.Position = 0; + output.SetLength(0); + input.CopyTo(output); + output.Position = 0; + + // This path updates the ext block bitmap, group descriptors, primary + backup + // superblocks and supported checksums in place. It never re-authors the inode tree. + _ = ExtInPlaceShrinker.ShrinkToFit(output); + output.Position = 0; + } + + /// + /// Reports the allocation unit that is safe to observe without pretending that a + /// block-size conversion is currently available for Lustre target metadata. + /// + public static LayoutAnalysis AnalyzeLayout(Stream image) { + ArgumentNullException.ThrowIfNull(image); + var geometry = ReadGeometry(image); + return new LayoutAnalysis { + ImageSize = image.Length, + CurrentUnitSize = geometry.BlockSize, + CurrentSlackBytes = 0, + OptimalUnitSize = geometry.BlockSize, + OptimalSlackBytes = 0, + RequiresRebuild = [], + Notes = [ + "Lustre ldiskfs block geometry can be inspected safely in place.", + "Block-size relayout is deliberately not offered: rebuilding as generic ext would discard Lustre LMA/LOV/FID xattrs and target metadata.", + ], + }; + } + + private static Geometry ReadGeometry(Stream image) { + if (!image.CanRead || !image.CanSeek) + throw new ArgumentException("Lustre ldiskfs maintenance requires a readable, seekable stream.", nameof(image)); + if (image.Length < SuperblockOffset + 1024) + throw new InvalidDataException("Lustre/ldiskfs: image is too small for an ext superblock."); + + var superblock = new byte[1024]; + image.Position = SuperblockOffset; + image.ReadExactly(superblock); + + var magic = BinaryPrimitives.ReadUInt16LittleEndian(superblock.AsSpan(56, 2)); + if (magic != ExtMagic) + throw new InvalidDataException($"Lustre maintenance requires an ldiskfs backing image (ext magic 0xEF53, found 0x{magic:X4})."); + + var logBlockSize = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(24, 4)); + if (logBlockSize > 6) + throw new InvalidDataException($"Lustre/ldiskfs: unsupported block-size exponent {logBlockSize}."); + var blockSize = checked(1024 << (int)logBlockSize); + + var firstDataBlock = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(20, 4)); + var blocksPerGroup = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(32, 4)); + if (blocksPerGroup == 0) + throw new InvalidDataException("Lustre/ldiskfs: s_blocks_per_group is zero."); + + var featureCompat = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(92, 4)); + var featureIncompat = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(96, 4)); + var featureRoCompat = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(100, 4)); + + ulong totalBlocks = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(4, 4)); + if ((featureIncompat & Incompat64Bit) != 0) + totalBlocks |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(0x150, 4)) << 32; + if (totalBlocks <= firstDataBlock) + throw new InvalidDataException("Lustre/ldiskfs: invalid block count."); + + var descriptorSize = 32; + if ((featureIncompat & Incompat64Bit) != 0) { + descriptorSize = BinaryPrimitives.ReadUInt16LittleEndian(superblock.AsSpan(0xFE, 2)); + if (descriptorSize < 32) descriptorSize = 32; + } + if (descriptorSize > blockSize) + throw new InvalidDataException($"Lustre/ldiskfs: group descriptor size {descriptorSize} exceeds block size {blockSize}."); + + var groupCount64 = (totalBlocks - firstDataBlock + blocksPerGroup - 1) / blocksPerGroup; + if (groupCount64 is 0 or > uint.MaxValue) + throw new InvalidDataException($"Lustre/ldiskfs: unsupported block-group count {groupCount64}."); + var groupCount = (uint)groupCount64; + + var descriptorTableOffset = checked((long)(firstDataBlock + 1UL) * blockSize); + var descriptorBytes = checked((long)groupCount * descriptorSize); + if (descriptorTableOffset < 0 || descriptorTableOffset + descriptorBytes > image.Length) + throw new InvalidDataException("Lustre/ldiskfs: primary group descriptor table is truncated."); + + if (totalBlocks > (ulong)(long.MaxValue / blockSize)) + throw new InvalidDataException("Lustre/ldiskfs: volume is too large for stream byte addressing."); + var declaredBytes = checked((long)totalBlocks * blockSize); + if (declaredBytes > image.Length) + throw new InvalidDataException($"Lustre/ldiskfs: superblock declares {declaredBytes} bytes but the image contains only {image.Length}."); + + return new Geometry( + blockSize, totalBlocks, firstDataBlock, blocksPerGroup, groupCount, + descriptorSize, descriptorTableOffset, featureCompat, featureIncompat, featureRoCompat); + } + + private static void ValidateWritableProfile(Geometry geometry) { + if ((geometry.FeatureIncompat & IncompatRecover) != 0) + throw new NotSupportedException("Lustre maintenance refuses an ldiskfs image that still needs journal recovery; replay it first."); + if ((geometry.FeatureIncompat & IncompatJournalDev) != 0) + throw new NotSupportedException("Lustre maintenance does not support ldiskfs images with an external journal device."); + if ((geometry.FeatureIncompat & IncompatMetaBg) != 0) + throw new NotSupportedException("Lustre maintenance does not yet support ext META_BG descriptor placement."); + if ((geometry.FeatureRoCompat & RoCompatBigalloc) != 0) + throw new NotSupportedException("Lustre maintenance does not yet support ext BIGALLOC cluster bitmaps."); + if ((geometry.FeatureRoCompat & RoCompatReadOnly) != 0) + throw new NotSupportedException("Lustre maintenance refuses an ext image explicitly marked read-only."); + } + + private static void ValidateShrinkProfile(Stream image, Geometry geometry) { + if (geometry.TotalBlocks > uint.MaxValue) + throw new NotSupportedException("Lustre shrink currently supports ldiskfs volumes up to UInt32 block counts."); + if ((geometry.FeatureCompat & CompatSparseSuper2) != 0) + throw new NotSupportedException("Lustre shrink does not yet rewrite SPARSE_SUPER2 backup locations."); + + var descriptor = new byte[geometry.DescriptorSize]; + for (uint group = 0; group < geometry.GroupCount; ++group) { + ReadDescriptor(image, geometry, group, descriptor); + var flags = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.AsSpan(0x12, 2)); + if ((flags & BgBlockUninit) != 0) + throw new NotSupportedException( + $"Lustre shrink refuses lazy BLOCK_UNINIT group {group}; its on-disk allocation bitmap is not authoritative yet."); + } + } + + private static void ReadDescriptor(Stream image, Geometry geometry, uint group, byte[] destination) { + var offset = checked(geometry.DescriptorTableOffset + (long)group * geometry.DescriptorSize); + if (offset + geometry.DescriptorSize > image.Length) + throw new InvalidDataException($"Lustre/ldiskfs: group descriptor {group} is truncated."); + image.Position = offset; + image.ReadExactly(destination); + } + + private static ulong ReadBlockBitmapBlock(ReadOnlySpan descriptor, int descriptorSize) { + ulong block = BinaryPrimitives.ReadUInt32LittleEndian(descriptor[..4]); + if (descriptorSize >= 64) + block |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(descriptor.Slice(0x20, 4)) << 32; + return block; + } + + private static long CheckedByteOffset(ulong block, int blockSize) { + if (block > (ulong)(long.MaxValue / blockSize)) + throw new InvalidDataException("Lustre/ldiskfs: block address exceeds stream addressing range."); + return checked((long)block * blockSize); + } + + private static void ZeroRange(Stream image, long offset, long length, byte[] zeros) { + image.Position = offset; + while (length > 0) { + var chunk = (int)Math.Min(length, zeros.Length); + image.Write(zeros, 0, chunk); + length -= chunk; + } + } +} From f38da73b7c1e18eeb4fc1d024f49085acc58e69a Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:06:15 +0200 Subject: [PATCH 074/165] Test MooseFS versioned metadata maintenance --- Compression.Tests/MooseFs/MooseFsTests.cs | 544 ++++++++++++---------- 1 file changed, 304 insertions(+), 240 deletions(-) diff --git a/Compression.Tests/MooseFs/MooseFsTests.cs b/Compression.Tests/MooseFs/MooseFsTests.cs index 153aec269..4cc821bdd 100644 --- a/Compression.Tests/MooseFs/MooseFsTests.cs +++ b/Compression.Tests/MooseFs/MooseFsTests.cs @@ -6,347 +6,411 @@ namespace Compression.Tests.MooseFs; /// -/// Tests for the MooseFS master-metadata partial R/O reader. -/// MooseFS itself is a distributed FS and we don't have golden cluster -/// images, so these tests use synthetic MooseFS-shaped images built byte by -/// byte against the documented envelope: 8-byte signature, 16-byte counters, -/// stream of (8-byte tag, 8-byte BE length, payload) sections, 16-byte EOF -/// marker. This validates the envelope walker without making any claim -/// about the version-specific NODE / EDGE / CHNK record bodies. +/// Tests for the MooseFS master-metadata envelope and the maintenance operations +/// that are safe on a standalone metadata.mfs image. Synthetic images contain +/// only specification-defined framing; NODE/EDGE/CHNK payloads remain opaque. /// [TestFixture] public class MooseFsTests { - private static byte[] BuildHeader(string signature = "MFSM 2.0", - ulong fileIdCounter = 100, - ulong metadataVersion = 42) { + private static byte[] BuildModernHeader( + string signature = "MFSM 2.0", + ulong metadataVersion = 42, + ulong metaId = 100) { if (signature.Length != 8) throw new ArgumentException("MooseFS signature must be exactly 8 ASCII bytes.", nameof(signature)); - var buf = new byte[8 + 16]; - Encoding.ASCII.GetBytes(signature).CopyTo(buf.AsSpan(0, 8)); - BinaryPrimitives.WriteUInt64BigEndian(buf.AsSpan(8, 8), fileIdCounter); - BinaryPrimitives.WriteUInt64BigEndian(buf.AsSpan(16, 8), metadataVersion); - return buf; + + var result = new byte[24]; + Encoding.ASCII.GetBytes(signature).CopyTo(result.AsSpan(0, 8)); + BinaryPrimitives.WriteUInt64BigEndian(result.AsSpan(8, 8), metadataVersion); + BinaryPrimitives.WriteUInt64BigEndian(result.AsSpan(16, 8), metaId); + return result; + } + + private static byte[] BuildPre20Header( + string signature = "MFSM 1.9", + uint maxNodeId = 123, + ulong metadataVersion = 42, + uint nextSessionId = 7) { + if (signature.Length != 8) + throw new ArgumentException("MooseFS signature must be exactly 8 ASCII bytes.", nameof(signature)); + + var result = new byte[24]; + Encoding.ASCII.GetBytes(signature).CopyTo(result.AsSpan(0, 8)); + BinaryPrimitives.WriteUInt32BigEndian(result.AsSpan(8, 4), maxNodeId); + BinaryPrimitives.WriteUInt64BigEndian(result.AsSpan(12, 8), metadataVersion); + BinaryPrimitives.WriteUInt32BigEndian(result.AsSpan(20, 4), nextSessionId); + return result; } private static byte[] BuildSection(string tag, byte[] payload) { if (tag.Length != 8) throw new ArgumentException("MooseFS section tag must be exactly 8 ASCII bytes.", nameof(tag)); - var buf = new byte[8 + 8 + payload.Length]; - Encoding.ASCII.GetBytes(tag).CopyTo(buf.AsSpan(0, 8)); - BinaryPrimitives.WriteUInt64BigEndian(buf.AsSpan(8, 8), (ulong)payload.Length); - payload.CopyTo(buf.AsSpan(16)); - return buf; + + var result = new byte[16 + payload.Length]; + Encoding.ASCII.GetBytes(tag).CopyTo(result.AsSpan(0, 8)); + BinaryPrimitives.WriteUInt64BigEndian(result.AsSpan(8, 8), (ulong)payload.Length); + payload.CopyTo(result.AsSpan(16)); + return result; } private static byte[] EofMarker() => "[MFS EOF MARKER]"u8.ToArray(); private static byte[] Concat(params byte[][] parts) { - var total = parts.Sum(p => p.Length); - var buf = new byte[total]; - var off = 0; - foreach (var p in parts) { - p.CopyTo(buf.AsSpan(off)); - off += p.Length; + var total = parts.Sum(static p => p.Length); + var result = new byte[total]; + var offset = 0; + foreach (var part in parts) { + part.CopyTo(result.AsSpan(offset)); + offset += part.Length; } - return buf; + return result; } - private static byte[] BuildMinimalImageWithSections(params (string Tag, int PayloadLen)[] sections) { - var parts = new List { BuildHeader() }; - foreach (var (tag, len) in sections) { - var payload = new byte[len]; - for (var i = 0; i < len; i++) payload[i] = (byte)(i & 0xFF); + private static byte[] BuildModernImage(params (string Tag, int PayloadLength)[] sections) { + var parts = new List { BuildModernHeader() }; + foreach (var (tag, length) in sections) { + var payload = new byte[length]; + for (var i = 0; i < payload.Length; ++i) + payload[i] = (byte)i; parts.Add(BuildSection(tag, payload)); } parts.Add(EofMarker()); return Concat(parts.ToArray()); } - // ============================================================ - // Descriptor sanity checks - // ============================================================ - [Test, Category("HappyPath")] public void Descriptor_IdentifiesByMagic() { - var d = new MooseFsFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("MooseFs")); - Assert.That(d.Extensions, Does.Contain(".mfsm")); - Assert.That(d.Extensions, Does.Not.Contain(".mfs"), - "MooseFS must not claim .mfs — it collides with the Macintosh File System (FileSystem.Mfs)."); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(1)); - Assert.That(d.MagicSignatures[0].Bytes, Is.EqualTo("MFSM"u8.ToArray())); + var descriptor = new MooseFsFormatDescriptor(); + Assert.That(descriptor.Id, Is.EqualTo("MooseFs")); + Assert.That(descriptor.Extensions, Does.Contain(".mfsm")); + Assert.That(descriptor.Extensions, Does.Not.Contain(".mfs"), + "MooseFS must not claim .mfs because it collides with Macintosh File System."); + Assert.That(descriptor.MagicSignatures, Has.Count.EqualTo(1)); + Assert.That(descriptor.MagicSignatures[0].Bytes, Is.EqualTo("MFSM"u8.ToArray())); } [Test, Category("HappyPath")] - public void Descriptor_DoesNotAdvertiseCreateOrModify() { - var d = new MooseFsFormatDescriptor(); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False, - "MooseFS metadata.mfs is not writable from a single image — chunks live on chunk servers."); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanList), Is.True); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanExtract), Is.True); + public void Descriptor_AdvertisesOnlyHonestStandaloneCapabilities() { + var descriptor = new MooseFsFormatDescriptor(); + Assert.Multiple(() => { + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanList), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanExtract), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + }); } [Test, Category("HappyPath")] public void Descriptor_DescriptionStatesChunkServerLimitation() { - var d = new MooseFsFormatDescriptor(); - var desc = d.Description.ToLowerInvariant(); - Assert.That(desc, Does.Contain("chunk server").Or.Contain("chunk-server").Or.Contain("chunkserver"), - "Description must be honest about file data living on chunk servers."); + var description = new MooseFsFormatDescriptor().Description.ToLowerInvariant(); + Assert.That(description, Does.Contain("chunk server")); + Assert.That(description, Does.Contain("not claimed")); } - // ============================================================ - // Reader — equivalence classes for the parser entry - // ============================================================ - [Test, Category("Boundary")] - public void Reader_RejectsImageSmallerThanHeader() { - using var ms = new MemoryStream(new byte[] { 0x4D, 0x46, 0x53 }); // "MFS", missing M - Assert.Throws(() => _ = new MooseFsReader(ms)); + public void Reader_RejectsImageSmallerThanSignature() { + using var stream = new MemoryStream("MFSM"u8.ToArray()); + Assert.Throws(() => _ = new MooseFsReader(stream)); } [Test, Category("Exception")] public void Reader_RejectsWrongMagic() { - var buf = new byte[64]; - Encoding.ASCII.GetBytes("NOPENOPE").CopyTo(buf.AsSpan(0, 8)); - using var ms = new MemoryStream(buf); - Assert.Throws(() => _ = new MooseFsReader(ms)); + using var stream = new MemoryStream("NOPENOPE"u8.ToArray()); + Assert.Throws(() => _ = new MooseFsReader(stream)); } [Test, Category("HappyPath")] - public void Reader_HeaderOnly_ParsesSignatureAndCounters() { - // Just header (8 bytes signature + 16 bytes counters) + EOF marker — - // no sections. ParseStatus must be "ok" since the EOF marker IS the - // legal end of the stream. - var image = Concat(BuildHeader(), EofMarker()); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.Signature, Is.EqualTo("MFSM 2.0")); - Assert.That(r.FileIdCounter, Is.EqualTo(100UL)); - Assert.That(r.MetadataVersion, Is.EqualTo(42UL)); - Assert.That(r.ParseStatus, Is.EqualTo("ok")); - Assert.That(r.Sections, Is.Empty); - Assert.That(r.Entries.Select(e => e.Name), - Is.EquivalentTo(new[] { "metadata.ini", "moosefs-master.bin" })); + public void Reader_ModernHeader_ParsesMetadataVersionAndMetaId() { + var image = Concat(BuildModernHeader(metadataVersion: 42, metaId: 100), EofMarker()); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.Multiple(() => { + Assert.That(reader.ValidHeader, Is.True); + Assert.That(reader.Signature, Is.EqualTo("MFSM 2.0")); + Assert.That(reader.FileFormatVersion, Is.EqualTo(0x20)); + Assert.That(reader.MetadataVersion, Is.EqualTo(42UL)); + Assert.That(reader.MetaId, Is.EqualTo(100UL)); + Assert.That(reader.MaxNodeId, Is.Null); + Assert.That(reader.NextSessionId, Is.Null); + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + Assert.That(reader.Sections, Is.Empty); + }); } [Test, Category("Boundary")] - public void Reader_TooShortForCounters_StaysHeaderOnly() { - // Only signature, no room for 16-byte counters or sections. - var image = "MFSM 2.0"u8.ToArray(); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.Signature, Is.EqualTo("MFSM 2.0")); - Assert.That(r.FileIdCounter, Is.Null); - Assert.That(r.MetadataVersion, Is.Null); - Assert.That(r.ParseStatus, Is.EqualTo("header-only")); - } - - // ============================================================ - // Reader — section index walk - // ============================================================ + public void Reader_TooShortForMetadataHeader_StaysHeaderOnly() { + using var stream = new MemoryStream("MFSM 2.0"u8.ToArray()); + using var reader = new MooseFsReader(stream); + Assert.Multiple(() => { + Assert.That(reader.FileFormatVersion, Is.EqualTo(0x20)); + Assert.That(reader.MetadataVersion, Is.Null); + Assert.That(reader.MetaId, Is.Null); + Assert.That(reader.ParseStatus, Is.EqualTo("header-only")); + }); + } [Test, Category("HappyPath")] - public void Reader_WalksAllSections_AndStopsAtEofMarker() { - var image = BuildMinimalImageWithSections( + public void Reader_MfsmNew_IsOfficialEmptyBootstrap() { + using var stream = new MemoryStream("MFSM NEW"u8.ToArray()); + using var reader = new MooseFsReader(stream); + Assert.Multiple(() => { + Assert.That(reader.ValidHeader, Is.True); + Assert.That(reader.IsEmptyBootstrap, Is.True); + Assert.That(reader.FileFormatVersion, Is.Null); + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + Assert.That(reader.Sections, Is.Empty); + }); + } + + [Test, Category("Exception")] + public void Reader_MfsmNewWithTrailingData_IsRejectedAsCanonicalImage() { + var image = Concat("MFSM NEW"u8.ToArray(), [0x42]); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + Assert.That(reader.ParseStatus, Is.EqualTo("trailing-data")); + } + + [Test, Category("HappyPath")] + public void Reader_WalksModernSections_AndStopsAtEofMarker() { + var image = BuildModernImage( ("SESS 1.0", 32), - ("STAT 1.0", 16), ("NODE 1.0", 256), ("EDGE 1.0", 128), ("CHNK 1.0", 64)); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + Assert.That(reader.Sections.Select(static s => s.Tag), + Is.EqualTo(new[] { "SESS 1.0", "NODE 1.0", "EDGE 1.0", "CHNK 1.0" })); + Assert.That(reader.Sections.Select(static s => s.Length), + Is.EqualTo(new long[] { 32, 256, 128, 64 })); + } + + [Test, Category("HappyPath")] + public void Reader_SectionedPre20Format_UsesLegacyHeaderShape() { + var image = Concat( + BuildPre20Header("MFSM 1.9", maxNodeId: 321, metadataVersion: 88, nextSessionId: 9), + BuildSection("NODE 1.0", new byte[16]), + EofMarker()); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.Multiple(() => { + Assert.That(reader.FileFormatVersion, Is.EqualTo(0x19)); + Assert.That(reader.MaxNodeId, Is.EqualTo(321U)); + Assert.That(reader.MetadataVersion, Is.EqualTo(88UL)); + Assert.That(reader.NextSessionId, Is.EqualTo(9U)); + Assert.That(reader.MetaId, Is.Null); + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + Assert.That(reader.Sections, Has.Count.EqualTo(1)); + }); + } - Assert.That(r.ParseStatus, Is.EqualTo("ok")); - Assert.That(r.Sections, Has.Count.EqualTo(5)); - Assert.That(r.Sections.Select(s => s.Tag), - Is.EqualTo(new[] { "SESS 1.0", "STAT 1.0", "NODE 1.0", "EDGE 1.0", "CHNK 1.0" })); - Assert.That(r.Sections.Select(s => s.Length), - Is.EqualTo(new long[] { 32, 16, 256, 128, 64 })); + [Test, Category("HappyPath")] + public void Reader_Pre16Format_RemainsOpaqueAndUsesZeroEofMarker() { + var opaqueLegacyBody = new byte[] { 0x01, 0xA5, 0x7F, 0x00, 0xCC }; + var image = Concat( + BuildPre20Header("MFSM 1.5", maxNodeId: 456, metadataVersion: 99, nextSessionId: 11), + opaqueLegacyBody, + new byte[16]); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.Multiple(() => { + Assert.That(reader.FileFormatVersion, Is.EqualTo(0x15)); + Assert.That(reader.MaxNodeId, Is.EqualTo(456U)); + Assert.That(reader.MetadataVersion, Is.EqualTo(99UL)); + Assert.That(reader.NextSessionId, Is.EqualTo(11U)); + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + Assert.That(reader.Sections, Is.Empty, + "Pre-1.6 bodies are not section-framed and must not be guessed from payload bytes."); + }); } [Test, Category("HappyPath")] - public void Reader_SurfacesPerSectionPayloadAsEntry() { - var image = BuildMinimalImageWithSections( - ("NODE 1.0", 100), - ("EDGE 1.0", 50)); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - - var names = r.Entries.Select(e => e.Name).ToList(); + public void Reader_SurfacesPerSectionPayloadAsSyntheticEntry() { + var image = BuildModernImage(("NODE 1.0", 100), ("EDGE 1.0", 50)); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + var names = reader.Entries.Select(static e => e.Name).ToArray(); Assert.That(names, Does.Contain("metadata.ini")); Assert.That(names, Does.Contain("moosefs-master.bin")); Assert.That(names, Does.Contain("section_NODE_1_0.bin")); Assert.That(names, Does.Contain("section_EDGE_1_0.bin")); - var node = r.Entries.First(e => e.Name == "section_NODE_1_0.bin"); - Assert.That(node.Size, Is.EqualTo(100)); - // First payload byte was set to (byte)(i & 0xFF) = 0 at i=0. - Assert.That(node.Data[0], Is.EqualTo(0)); + var node = reader.Entries.First(static e => e.Name == "section_NODE_1_0.bin"); Assert.That(node.Data[99], Is.EqualTo(99)); } [Test, Category("Exception")] public void Reader_TruncatedSectionLength_MarksTruncated() { - // Build header + a section that claims more bytes than the image actually has. - var header = BuildHeader(); - var tag = "NODE 1.0"u8.ToArray(); - var lenBuf = new byte[8]; - BinaryPrimitives.WriteUInt64BigEndian(lenBuf, 10_000UL); // claim 10k bytes - var shortPayload = new byte[100]; // only 100 actually present - var image = Concat(header, tag, lenBuf, shortPayload); - - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.ParseStatus, Is.EqualTo("truncated")); - Assert.That(r.Sections, Is.Empty, - "A length-overflow section must not be added to the section list."); + var length = new byte[8]; + BinaryPrimitives.WriteUInt64BigEndian(length, 10_000UL); + var image = Concat(BuildModernHeader(), "NODE 1.0"u8.ToArray(), length, new byte[100]); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.That(reader.ParseStatus, Is.EqualTo("truncated")); + Assert.That(reader.Sections, Is.Empty); } [Test, Category("Exception")] public void Reader_NoEofMarker_MarksTruncated() { - // Valid sections but no [MFS EOF MARKER] trailing. - var header = BuildHeader(); - var sec = BuildSection("NODE 1.0", new byte[64]); - var image = Concat(header, sec); // no EOF marker - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ParseStatus, Is.EqualTo("truncated")); - Assert.That(r.Sections, Has.Count.EqualTo(1), - "The walker should still surface every successfully parsed section before noticing the missing EOF marker."); + var image = Concat(BuildModernHeader(), BuildSection("NODE 1.0", new byte[64])); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.That(reader.ParseStatus, Is.EqualTo("truncated")); + Assert.That(reader.Sections, Has.Count.EqualTo(1)); } [Test, Category("Exception")] - public void Reader_NonAsciiSectionTag_MarksTruncated() { - // Image with a "section tag" that's actually binary garbage — walker - // must reject it rather than accepting random bytes as a tag. - var header = BuildHeader(); - var garbageTag = new byte[] { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 }; - var lenBuf = new byte[8]; - BinaryPrimitives.WriteUInt64BigEndian(lenBuf, 0UL); - var image = Concat(header, garbageTag, lenBuf, EofMarker()); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ParseStatus, Is.EqualTo("truncated")); - Assert.That(r.Sections, Is.Empty); - } - - // ============================================================ - // metadata.ini content - // ============================================================ + public void Reader_EofMarkerMustBeLastBytes() { + var image = Concat(BuildModernHeader(), EofMarker(), [0xDE, 0xAD]); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + Assert.That(reader.ParseStatus, Is.EqualTo("trailing-data")); + } - [Test, Category("HappyPath")] - public void MetadataIni_ReportsHeaderAndSectionTable() { - var image = BuildMinimalImageWithSections( - ("NODE 1.0", 100), - ("EDGE 1.0", 50)); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); + [Test, Category("Exception")] + public void Reader_InvalidSectionTag_MarksTruncated() { + var invalidTag = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 }; + var image = Concat(BuildModernHeader(), invalidTag, new byte[8], EofMarker()); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.That(reader.ParseStatus, Is.EqualTo("truncated")); + Assert.That(reader.Sections, Is.Empty); + } - var ini = r.Entries.First(e => e.Name == "metadata.ini"); - var text = Encoding.UTF8.GetString(ini.Data); + [Test, Category("HappyPath")] + public void MetadataIni_ReportsCorrectVersionedHeaderAndSectionTable() { + var image = BuildModernImage(("NODE 1.0", 100), ("EDGE 1.0", 50)); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + var text = Encoding.UTF8.GetString(reader.Entries.First(static e => e.Name == "metadata.ini").Data); Assert.That(text, Does.Contain("parse_status=ok")); Assert.That(text, Does.Contain("signature=MFSM 2.0")); - Assert.That(text, Does.Contain("magic_tag=MFSM")); - Assert.That(text, Does.Contain("file_id_counter=100")); + Assert.That(text, Does.Contain("file_format_version=2.0")); Assert.That(text, Does.Contain("metadata_version=42")); + Assert.That(text, Does.Contain("meta_id=100")); + Assert.That(text, Does.Not.Contain("file_id_counter")); Assert.That(text, Does.Contain("section_count=2")); Assert.That(text, Does.Contain("NODE 1.0")); - Assert.That(text, Does.Contain("EDGE 1.0")); - Assert.That(text.ToLowerInvariant(), Does.Contain("chunk server")); + Assert.That(text.ToLowerInvariant(), Does.Contain("chunk servers")); } - [Test, Category("HappyPath")] - public void MetadataIni_HeaderOnly_FlagsHeaderOnlyStatus() { - // Bare 8-byte signature, no counters or sections at all. - var image = "MFSM 2.0"u8.ToArray(); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - var ini = r.Entries.First(e => e.Name == "metadata.ini"); - var text = Encoding.UTF8.GetString(ini.Data); - Assert.That(text, Does.Contain("parse_status=header-only")); - Assert.That(text, Does.Contain("section_count=0")); - } - - // ============================================================ - // Descriptor.List / Extract / OpenEntry — public surface - // ============================================================ - [Test, Category("HappyPath")] public void Descriptor_List_ReturnsMetadataPlusSections() { - var d = new MooseFsFormatDescriptor(); - var image = BuildMinimalImageWithSections(("NODE 1.0", 32)); - using var ms = new MemoryStream(image); - var entries = d.List(ms, password: null); - var names = entries.Select(e => e.Name).ToList(); + var descriptor = new MooseFsFormatDescriptor(); + using var stream = new MemoryStream(BuildModernImage(("NODE 1.0", 32))); + var names = descriptor.List(stream, password: null).Select(static e => e.Name).ToArray(); + Assert.That(names, Does.Contain("metadata.ini")); Assert.That(names, Does.Contain("moosefs-master.bin")); Assert.That(names, Does.Contain("section_NODE_1_0.bin")); } [Test, Category("HappyPath")] - public void Descriptor_Extract_WritesAllSurfacedEntries() { - var d = new MooseFsFormatDescriptor(); - var image = BuildMinimalImageWithSections(("NODE 1.0", 32)); - var tempDir = Path.Combine(Path.GetTempPath(), "moosefs-extract-" + Guid.NewGuid().ToString("N")[..8]); - Directory.CreateDirectory(tempDir); - try { - using var ms = new MemoryStream(image); - d.Extract(ms, tempDir, password: null, files: null); - - Assert.That(File.Exists(Path.Combine(tempDir, "metadata.ini")), Is.True); - Assert.That(File.Exists(Path.Combine(tempDir, "moosefs-master.bin")), Is.True); - Assert.That(File.Exists(Path.Combine(tempDir, "section_NODE_1_0.bin")), Is.True); - } finally { - try { Directory.Delete(tempDir, recursive: true); } catch { /* best effort */ } - } + public void Descriptor_OpenEntry_ReturnsExactlySectionPayload() { + var operations = (IArchiveFormatOperations)new MooseFsFormatDescriptor(); + using var archive = new MemoryStream(BuildModernImage(("NODE 1.0", 100))); + using var entry = operations.OpenEntry(archive, "section_NODE_1_0.bin", password: null); + + Assert.That(entry.Length, Is.EqualTo(100)); + var data = new byte[200]; + Assert.That(entry.Read(data, 0, data.Length), Is.EqualTo(100)); + } + + [Test, Category("Exception")] + public void Descriptor_OpenEntry_UnknownName_Throws() { + var operations = (IArchiveFormatOperations)new MooseFsFormatDescriptor(); + using var archive = new MemoryStream(BuildModernImage(("NODE 1.0", 32))); + Assert.Throws(() => operations.OpenEntry(archive, "does-not-exist.bin", null)); } [Test, Category("HappyPath")] - public void Descriptor_OpenEntry_ReturnsBoundedStream() { - var d = (IArchiveFormatOperations)new MooseFsFormatDescriptor(); - var image = BuildMinimalImageWithSections(("NODE 1.0", 100)); - using var ms = new MemoryStream(image); - using var s = d.OpenEntry(ms, "section_NODE_1_0.bin", password: null); - Assert.That(s.Length, Is.EqualTo(100)); - var buf = new byte[200]; - var n = s.Read(buf, 0, buf.Length); - Assert.That(n, Is.EqualTo(100), - "Bounded entry stream must yield exactly the section payload, never adjacent bytes."); + public void Descriptor_Layout_CoversEveryByteWithoutInventingFreeSpace() { + var image = BuildModernImage(("NODE 1.0", 32), ("EDGE 1.0", 17)); + var layout = (IArchiveLayoutMap)new MooseFsFormatDescriptor(); + using var archive = new MemoryStream(image); + var blocks = layout.EnumerateLayout(archive).ToArray(); + + Assert.Multiple(() => { + Assert.That(blocks, Is.Not.Empty); + Assert.That(blocks.Sum(static block => block.Length), Is.EqualTo(image.LongLength)); + Assert.That(blocks.Any(static block => block.Kind == DefragBlockKind.Free), Is.False); + Assert.That(blocks[0].Offset, Is.Zero); + Assert.That(blocks[^1].Offset + blocks[^1].Length, Is.EqualTo(image.LongLength)); + }); + } + + [Test, Category("HappyPath")] + public void Descriptor_LegacyLayout_IsConservativelyAllMetadata() { + var image = Concat(BuildPre20Header("MFSM 1.5"), [0x01, 0x02, 0x03], new byte[16]); + var layout = (IArchiveLayoutMap)new MooseFsFormatDescriptor(); + using var archive = new MemoryStream(image); + var blocks = layout.EnumerateLayout(archive).ToArray(); + + Assert.That(blocks, Has.Length.EqualTo(1)); + Assert.That(blocks[0].Offset, Is.Zero); + Assert.That(blocks[0].Length, Is.EqualTo(image.LongLength)); + Assert.That(blocks[0].Kind, Is.EqualTo(DefragBlockKind.MetadataReserved)); } [Test, Category("Exception")] - public void Descriptor_OpenEntry_UnknownName_Throws() { - var d = (IArchiveFormatOperations)new MooseFsFormatDescriptor(); - var image = BuildMinimalImageWithSections(("NODE 1.0", 32)); - using var ms = new MemoryStream(image); - Assert.Throws(() => d.OpenEntry(ms, "does-not-exist.bin", password: null)); + public void Descriptor_Layout_FailsClosedForTruncatedImage() { + var image = Concat(BuildModernHeader(), BuildSection("NODE 1.0", new byte[8])); + var layout = (IArchiveLayoutMap)new MooseFsFormatDescriptor(); + using var archive = new MemoryStream(image); + Assert.That(layout.EnumerateLayout(archive), Is.Empty); } - // ============================================================ - // Pre-1.6 signature path (no counters) - // ============================================================ + [Test, Category("HappyPath")] + public void Descriptor_Wipe_IsNoOpForPackedMetadata() { + var original = BuildModernImage(("NODE 1.0", 64), ("EDGE 1.0", 23)); + using var archive = new MemoryStream(); + archive.Write(original); + archive.Position = 0; + + var wiped = ((IWipeEmpty)new MooseFsFormatDescriptor()).WipeUnusedSpace(archive); + + Assert.That(wiped, Is.Zero); + Assert.That(archive.ToArray(), Is.EqualTo(original)); + } [Test, Category("HappyPath")] - public void Reader_LegacyMfsm15Signature_SkipsCounterBlock() { - // For "MFSM 1.4" / "MFSM 1.5", section stream starts immediately after - // the 8-byte signature — there are no counter bytes. We build a minimal - // image and confirm the walker picks up sections from offset 8. - var sig = "MFSM 1.5"u8.ToArray(); - var sec = BuildSection("NODE 1.0", new byte[16]); - var image = Concat(sig, sec, EofMarker()); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.Signature, Is.EqualTo("MFSM 1.5")); - Assert.That(r.ParseStatus, Is.EqualTo("ok")); - Assert.That(r.Sections, Has.Count.EqualTo(1)); - Assert.That(r.Sections[0].Tag, Is.EqualTo("NODE 1.0")); - Assert.That(r.Sections[0].Length, Is.EqualTo(16)); + public void Descriptor_Purge_ResetsToOfficialEmptyBootstrap() { + using var archive = new MemoryStream(); + archive.Write(BuildModernImage(("NODE 1.0", 64), ("EDGE 1.0", 23))); + archive.Position = 0; + + ((IArchivePurgeable)new MooseFsFormatDescriptor()).Purge(archive); + + Assert.That(archive.ToArray(), Is.EqualTo("MFSM NEW"u8.ToArray())); + archive.Position = 0; + using var reader = new MooseFsReader(archive); + Assert.That(reader.IsEmptyBootstrap, Is.True); + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + } + + [Test, Category("Exception")] + public void Descriptor_Purge_RejectsStructurallyInvalidMetadata() { + using var archive = new MemoryStream(); + archive.Write(Concat(BuildModernHeader(), BuildSection("NODE 1.0", new byte[8]))); + archive.Position = 0; + + Assert.Throws(() => + ((IArchivePurgeable)new MooseFsFormatDescriptor()).Purge(archive)); } } From 3e07674bab9d6be6a4d469500c05e113f099b7e6 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:06:16 +0200 Subject: [PATCH 075/165] Reference shared NWFS engine from NWFS386 --- FileSystems/FileSystem.Nwfs386/FileSystem.Nwfs386.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/FileSystems/FileSystem.Nwfs386/FileSystem.Nwfs386.csproj b/FileSystems/FileSystem.Nwfs386/FileSystem.Nwfs386.csproj index 5f028fe08..333d0d1ef 100644 --- a/FileSystems/FileSystem.Nwfs386/FileSystem.Nwfs386.csproj +++ b/FileSystems/FileSystem.Nwfs386/FileSystem.Nwfs386.csproj @@ -6,6 +6,7 @@ + From 71d3b19d045b23368ee8eec8e592426591159b83 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:06:17 +0200 Subject: [PATCH 076/165] Fix T64 maintenance test typing --- Compression.Tests/T64/T64MaintenanceTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Compression.Tests/T64/T64MaintenanceTests.cs b/Compression.Tests/T64/T64MaintenanceTests.cs index 4661f0eb5..ecf6bc5e7 100644 --- a/Compression.Tests/T64/T64MaintenanceTests.cs +++ b/Compression.Tests/T64/T64MaintenanceTests.cs @@ -104,10 +104,10 @@ public void WipeUnusedSpace_ZerosFreeDirectorySlotAndPayloadGaps() { Assert.Multiple(() => { Assert.That(wiped, Is.EqualTo(32 + 4 + 3)); - Assert.That(raw.AsSpan(96, 32).ToArray(), Is.All.Zero); - Assert.That(raw.AsSpan(128, 4).ToArray(), Is.All.Zero); + Assert.That(raw.AsSpan(96, 32).ToArray(), Is.All.EqualTo(0)); + Assert.That(raw.AsSpan(128, 4).ToArray(), Is.All.EqualTo(0)); Assert.That(raw.AsSpan(132, payload.Length).ToArray(), Is.EqualTo(payload)); - Assert.That(raw.AsSpan(132 + payload.Length, 3).ToArray(), Is.All.Zero); + Assert.That(raw.AsSpan(132 + payload.Length, 3).ToArray(), Is.All.EqualTo(0)); }); } } @@ -182,7 +182,7 @@ private static (MemoryStream Stream, byte[] Payload) BuildGappedImage() { image[64] = 1; image[65] = 0x81; BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(66), 0x2000); - BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(68), 0x2000 + payload.Length); + BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(68), (ushort)(0x2000 + payload.Length)); BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(72), payloadOffset); "GAPTEST"u8.CopyTo(image.AsSpan(80)); image.AsSpan(87, 9).Fill(0x20); From 052fca6c845a076765998e268e6ac1622a904b69 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:06:34 +0200 Subject: [PATCH 077/165] + expose safe Lustre ldiskfs maintenance capabilities --- .../LustreFormatDescriptor.cs | 73 ++++++++++++++----- 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/FileSystems/FileSystem.Lustre/LustreFormatDescriptor.cs b/FileSystems/FileSystem.Lustre/LustreFormatDescriptor.cs index cf9221bf6..2a65e6087 100644 --- a/FileSystems/FileSystem.Lustre/LustreFormatDescriptor.cs +++ b/FileSystems/FileSystem.Lustre/LustreFormatDescriptor.cs @@ -6,27 +6,38 @@ namespace FileSystem.Lustre; /// -/// R/O descriptor for Lustre MDT/OST images via ldiskfs (ext4-compatible) -/// reader delegation. Surfaces the ldiskfs view of a single MDT or OST -/// backing store — NOT the Lustre logical view (which would require combining -/// MDT inode metadata with file data striped across multiple OSTs, out of -/// scope without live cluster metadata). +/// Descriptor for Lustre MDT/OST images backed by ldiskfs (ext4-compatible). +/// The archive projection surfaces one target's backing-store namespace — NOT the +/// distributed Lustre logical view, which requires correlating MDT metadata with +/// objects striped across multiple OSTs. +/// +/// +/// Maintenance is intentionally narrower than generic ext editing. Free-space wipe +/// follows the ldiskfs block allocation bitmap, and shrink updates ext geometry in +/// place, preserving every surviving allocated block. Add/replace/remove, defrag and +/// structural relayout are not advertised because rebuilding through the generic ext +/// writer would discard Lustre LMA/LOV/FID extended attributes and target metadata. +/// Legacy "LUSTRE" / "LUst" object-header dumps remain inspection-only. +/// /// /// Detection is extension-routed (.lustre / .ost / .mdt) and the legacy /// "LUSTRE" / "LUst" object-header magic at offset 0; ext4 superblock magic is -/// deliberately NOT registered here (it would steal detection from generic ext4 -/// images). When opened with an ldiskfs MDT/OST image (recognised by the .ost / -/// .mdt / .lustre extension), delegates the file walk -/// to FileSystem.Ext.ExtReader. +/// deliberately NOT registered here because that would steal generic ext4 images. /// /// References: /// -/// https://www.lustre.org/ — project home -/// https://wiki.lustre.org/ — Lustre wiki (architecture, ldiskfs/MDT/OST layout) -/// https://en.wikipedia.org/wiki/Lustre_(file_system) — Wikipedia article +/// https://wiki.lustre.org/Configuring_the_Lustre_File_System — mkfs.lustre examples showing ldiskfs backing targets +/// https://wiki.lustre.org/Understanding_Lustre_Internals — backing ldiskfs inspection and Lustre target internals +/// https://docs.kernel.org/filesystems/ext4/bitmaps.html — ext4 allocation bitmap semantics and BLOCK_UNINIT warning +/// https://docs.kernel.org/filesystems/ext4/group_descr.html — group descriptor and bitmap locations /// /// -public sealed class LustreFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { +public sealed class LustreFormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IWipeEmpty, + IArchiveShrinkable, + ILayoutOptimizable { /// /// Gets the id. @@ -86,12 +97,14 @@ public sealed class LustreFormatDescriptor : IFormatDescriptor, IArchiveFormatOp /// Gets the description. /// public string Description => - "Lustre R/O via ldiskfs (ext4) reader delegation. Surfaces the ldiskfs view of one " + - "MDT or OST backing store (file walk over the ext4-compatible block layout); Lustre " + - "xattrs (LMA, LOV EA striping, FID) are preserved in the raw image but not interpreted. " + - "The Lustre logical view (combining MDT inode metadata with file data striped across " + - "multiple OSTs) requires live cluster metadata and is out of scope. Legacy 'LUSTRE'/'LUst' " + - "object-header dumps still surface as raw bytes + metadata.ini."; + "Lustre target inspection via ldiskfs (ext4) reader delegation, with conservative " + + "offline maintenance on genuine ldiskfs images: bitmap-proven free-block wipe and " + + "in-place trailing-free shrink/compact. Lustre xattrs (LMA, LOV EA striping, FID) " + + "remain opaque and are preserved because maintenance never reconstructs live inodes. " + + "Add/remove/purge, defrag and block-size relayout are intentionally not claimed: a " + + "generic ext rebuild would lose Lustre target semantics. The distributed logical view " + + "still requires live cluster metadata and is out of scope. Legacy 'LUSTRE'/'LUst' dumps " + + "remain raw inspection-only objects."; /// /// Lists the entries in the supplied container. @@ -123,4 +136,26 @@ Stream IArchiveFormatOperations.OpenEntry(Stream archive, string entryName, stri var data = r.Extract(entry); return new BoundedEntryStream(new MemoryStream(data, writable: false), data.Length, leaveOpen: false); } + + /// + /// Zeroes complete ldiskfs blocks whose initialized allocation bitmap proves them free. + /// Cluster-tip and deleted-dirent requests are deliberately ignored: those byte ranges + /// cannot be proven dead without interpreting Lustre-specific inode/xattr semantics. + /// + public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) + => LustreLdiskfsMaintenance.WipeFreeBlocks(image); + + /// + /// Shrinks an offline ldiskfs target by trimming trailing free blocks while preserving + /// all surviving allocated blocks and Lustre-specific inode/xattr bytes. + /// + public void Shrink(Stream input, Stream output) + => LustreLdiskfsMaintenance.Shrink(input, output); + + /// + /// Reports the current ldiskfs block geometry. Structural relayout is deliberately not + /// offered until a Lustre-aware writer can preserve all target metadata. + /// + public LayoutAnalysis AnalyzeLayout(Stream image) + => LustreLdiskfsMaintenance.AnalyzeLayout(image); } From d5ae23066f775d5ea79cc5d41da66368309e6914 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:07:06 +0200 Subject: [PATCH 078/165] Update Intel HEX support matrix capabilities --- Hawkynt.FileFormats.FileSystems/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index d5725e500..063577775 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -64,7 +64,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [DMG](https://en.wikipedia.org/wiki/Apple_Disk_Image) | `Dmg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | — | [Apple disk images](https://developer.apple.com/library/archive/documentation/DeveloperTools/Conceptual/SoftwareDistribution4/Concepts/sd_disk_images.html) | | [EnCase EWF (E01)](https://en.wikipedia.org/wiki/EnCase#Expert_Witness_File_Format) | `Ewf` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | EnCase E01 segment chain | [libewf documentation](https://github.com/libyal/libewf/tree/main/documentation) | | [Flattened Device Tree Blob](https://en.wikipedia.org/wiki/Devicetree) | `Dtb` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Flattened Device Tree walked as a pseudo-archive | [Devicetree specification](https://www.devicetree.org/specifications/) | -| [Intel HEX](https://en.wikipedia.org/wiki/Intel_HEX) | `IntelHex` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | Intel HEX and Motorola S-record, normalised to `firmware.bin` + metadata; the writer takes one payload and the base and start addresses from `metadata.ini` | [Intel HEX](https://www.keil.com/support/docs/1584/_hlp_hexfile.htm) / [S-record](https://srecord.sourceforge.net/man/man5/srec_motorola.5.html) | +| [Intel HEX](https://en.wikipedia.org/wiki/Intel_HEX) | `IntelHex` | R/W | ✅ | ✅ rebuild | — | — | — | ✅ | own reader + struct-parity tests | Intel HEX and Motorola S-record, normalised to `firmware.bin` + metadata; the writer takes one payload and the base and start addresses from `metadata.ini` | [Intel HEX](https://www.keil.com/support/docs/1584/_hlp_hexfile.htm) / [S-record](https://srecord.sourceforge.net/man/man5/srec_motorola.5.html) | | [MDF/MDS](https://en.wikipedia.org/wiki/Alcohol_120%25) | `Mdf` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Alcohol 120%](https://en.wikipedia.org/wiki/Alcohol_120%25) | | [Nintendo PartitionFS](https://en.wikipedia.org/wiki/Nintendo_Switch) | `Pfs0` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | Nintendo Switch PartitionFS | [PFS0 (switchbrew)](https://switchbrew.org/wiki/NCA#PFS0) | | [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | @@ -344,7 +344,7 @@ cwb recover raw.img --mode files --format Jpeg,Png If this project saves you time or money, consider supporting its development: [![GitHub Sponsors](https://img.shields.io/badge/GitHub-Sponsor-EA4AAA?logo=githubsponsors)](https://github.com/sponsors/Hawkynt) -[![PayPal](https://img.shields.io/badge/PayPal-Donate-00457C?logo=paypal)](https://www.paypal.me/hawkynt) +[![PayPal](https://img.shields.io/badge/PayPal-Donate-00457C)](https://www.paypal.me/hawkynt) ## 📜 License From 3d682a8347025dd4dd6d996f5267fa9725aca571 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:07:34 +0200 Subject: [PATCH 079/165] * describe Lustre backing-store maintenance scope --- FileSystems/FileSystem.Lustre/LustreReader.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/FileSystems/FileSystem.Lustre/LustreReader.cs b/FileSystems/FileSystem.Lustre/LustreReader.cs index fb5b13e14..5b5651f75 100644 --- a/FileSystems/FileSystem.Lustre/LustreReader.cs +++ b/FileSystems/FileSystem.Lustre/LustreReader.cs @@ -193,9 +193,10 @@ private byte[] BuildLdiskfsMetadata() { bldr.Append(CultureInfo.InvariantCulture, $"image_size={_data.Length}\n"); bldr.Append(CultureInfo.InvariantCulture, $"ldiskfs_volume_label={this.LdiskfsVolumeLabel}\n"); bldr.Append(CultureInfo.InvariantCulture, $"ldiskfs_entry_count={(_extReader?.Entries.Count ?? 0)}\n"); - bldr.Append("note=R/O via ext4 reader delegation. Surfaces the ldiskfs view of a single MDT or OST "); - bldr.Append("backing store, NOT the Lustre logical view. Lustre xattrs (LMA, LOV EA striping, FID) "); - bldr.Append("are preserved in the raw image but not interpreted. Combining MDT inode metadata with "); + bldr.Append("note=Read projection via ext4 delegation. Conservative offline maintenance may wipe bitmap-proven "); + bldr.Append("free blocks and trim trailing free ldiskfs blocks without reconstructing live inodes. The view is one "); + bldr.Append("MDT or OST backing store, NOT the Lustre logical view. Lustre xattrs (LMA, LOV EA striping, FID) "); + bldr.Append("remain opaque and allocated metadata is preserved byte-for-byte. Combining MDT inode metadata with "); bldr.Append("file data striped across multiple OSTs requires live cluster metadata and is out-of-scope.\n"); return Encoding.UTF8.GetBytes(bldr.ToString()); } From 12ced657ff44b2798b5a039625571ee54d268ca7 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:07:54 +0200 Subject: [PATCH 080/165] * refresh IPSW maintenance support matrix --- Hawkynt.FileFormats.FileSystems/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index d5725e500..d76349136 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -57,7 +57,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | Format | Id | State | Compact | Defrag | Wipe | Shrink | Layout | Purge | Proof | Notes | Reference | | --- | --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | --- | --- | --- | -| [Apple IPSW](https://en.wikipedia.org/wiki/IPSW) | `Ipsw` | R/W | — | — | ✅ | — | — | ✅ | own reader + struct-parity tests | ZIP-based firmware archive | [IPSW](https://en.wikipedia.org/wiki/IPSW) | +| [Apple IPSW](https://en.wikipedia.org/wiki/IPSW) | `Ipsw` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | ZIP-based firmware archive | [IPSW](https://en.wikipedia.org/wiki/IPSW) | | [BIN/CUE](https://en.wikipedia.org/wiki/Cue_sheet_(computing)) | `BinCue` | R/W | — | — | ✅ | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [CUE sheet](https://wiki.hydrogenaud.io/index.php?title=Cue_sheet) | | [CDI](https://en.wikipedia.org/wiki/DiscJuggler) | `Cdi` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [DiscJuggler](https://en.wikipedia.org/wiki/DiscJuggler) | | [Commodore Lynx archive](https://en.wikipedia.org/wiki/Commodore_64) | `Lynx` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 Lynx archive | [Lynx](https://ist.uwaterloo.ca/~schepers/formats/LNX.TXT) | @@ -112,7 +112,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [ext2/3/4](https://en.wikipedia.org/wiki/Ext4) | `Ext` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`e2fsck -fn`](https://github.com/tytso/e2fsprogs) / `dumpe2fs` / `debugfs` + reverse `mke2fs` | ext2 / ext3 / ext4 | [ext4 documentation](https://www.kernel.org/doc/html/latest/filesystems/ext4/) | | [F2FS](https://en.wikipedia.org/wiki/F2FS) | `F2fs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `f2fs` mount r/w + [`fsck.f2fs`](https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs-tools.git) | — | [F2FS documentation](https://www.kernel.org/doc/html/latest/filesystems/f2fs.html) | | [GFS (Sistina/Red Hat, original)](https://en.wikipedia.org/wiki/GFS2) | `Gfs1` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; the Linux `gfs2` driver rejects GFS1 | Sistina GFS | [GFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | -| [GFS2 (Global File System 2)](https://en.wikipedia.org/wiki/GFS2) | `Gfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.gfs2 -n`](https://pagure.io/gfs2-utils) clean on populated create and after add/replace/remove; reverse `mkfs.gfs2` reader coverage | Standalone `lock_nolock`; stuffed + indirect-tree files; edits rebuild while preserving image-size floor and lock table | [GFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | +| [GFS2 (Global File System 2)](https://en.wikipedia.org/wiki/GFS2) | `Gfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.gfs2 -n`](https://pagure.io/gfs2-utils) + `debugfs.gfs2`; `mkfs.gfs2` image read by the package | Standalone `lock_nolock`; stuffed + indirect-tree files; edits rebuild while preserving image-size floor and lock table | [GFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | | [HAMMER (DragonFly BSD)](https://en.wikipedia.org/wiki/HAMMER_(file_system)) | `Hammer` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | DragonFly kernel mount r/w under QEMU + [`hammer show` / `hammer checkmap`](https://www.dragonflybsd.org/hammer/) | Single volume, about 1 GB minimum; edits rebuild the volume | [HAMMER](https://www.dragonflybsd.org/hammer/) | | [HAMMER2 (DragonFly BSD)](https://en.wikipedia.org/wiki/HAMMER2) | `Hammer2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | DragonFly kernel mount r/w under QEMU | — | [HAMMER2](https://www.dragonflybsd.org/hammer/) | | [HTFS (SCO High Throughput File System)](https://en.wikipedia.org/wiki/High_Throughput_File_System) | `Htfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | SCO HTFS | [HTFS](https://en.wikipedia.org/wiki/High_Throughput_File_System) | From a46538dbeb44ed089908f94a51e66e0ef58e16fb Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:07:59 +0200 Subject: [PATCH 081/165] Infer GlusterFS brick root from GFID index --- .../FileSystem.GlusterFs/GlusterFsReader.cs | 99 +++++++++++++++---- 1 file changed, 81 insertions(+), 18 deletions(-) diff --git a/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs b/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs index 44dc4d78a..6b25fdc42 100644 --- a/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs +++ b/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs @@ -19,16 +19,21 @@ namespace FileSystem.GlusterFs; /// intentionally exposes only the physical view of one supplied backing image. /// It does not claim to reconstruct a Gluster volume from one brick. /// +/// When the backing namespace contains Gluster's .glusterfs GFID +/// index, its parent directory identifies the brick root. Entries outside that +/// subtree and the index itself are omitted from the normal view. If no index is +/// present, the filesystem root is used as a conservative explicit-input +/// fallback. +/// /// The backing readers currently do not interpret or mutate Gluster xattrs. -/// Those bytes remain untouched because this type is read-only. The internal -/// .glusterfs GFID index is omitted from the normal brick listing so it is -/// not mistaken for user namespace content. +/// Those bytes remain untouched because this type is read-only. /// public sealed class GlusterFsReader : IDisposable { private const int ExtSuperblockOffset = 1024; private const int ExtMagicOffset = ExtSuperblockOffset + 56; private const ushort ExtMagic = 0xEF53; + private const string GlusterIndexName = ".glusterfs"; private readonly Stream _image; private readonly bool _ownsImage; @@ -51,6 +56,16 @@ public sealed class GlusterFsReader : IDisposable { /// public string BackingFileSystem { get; private set; } = ""; + /// + /// Gets the inferred brick root inside the backing filesystem. Empty means filesystem root. + /// + public string BrickRoot { get; private set; } = ""; + + /// + /// Gets whether a .glusterfs GFID index was present and used to locate the brick root. + /// + public bool HasGlusterIndex { get; private set; } + /// /// Initializes a reader over one brick backing-store image. /// @@ -107,16 +122,17 @@ private void ParseXfs() { _xfsReader = new XfsReader(_image, leaveOpen: true); this.BackingFileSystem = "xfs"; this.ValidHeader = true; + SetBrickRoot(_xfsReader.Entries.Select(entry => entry.Name)); AddMetadata(_xfsReader.Entries.Count); foreach (var entry in _xfsReader.Entries) { - if (IsGlusterInternal(entry.Name)) continue; + if (!TryGetBrickRelativePath(entry.Name, out var relativePath)) continue; var captured = entry; _entries.Add(new GlusterFsEntry { - Name = BrickPath(entry.Name), + Name = BrickPath(relativePath), Size = entry.Size, IsDirectory = entry.IsDirectory, - DataFactory = entry.IsDirectory ? null : () => _xfsReader.Extract(captured), + DataFactory = entry.IsDirectory ? null : () => _xfsReader!.Extract(captured), }); } } @@ -126,20 +142,64 @@ private void ParseExt() { _extReader = new ExtReader(_image, leaveOpen: true); this.BackingFileSystem = "ext"; this.ValidHeader = true; + SetBrickRoot(_extReader.Entries.Select(entry => entry.Name)); AddMetadata(_extReader.Entries.Count); foreach (var entry in _extReader.Entries) { - if (IsGlusterInternal(entry.Name)) continue; + if (!TryGetBrickRelativePath(entry.Name, out var relativePath)) continue; var captured = entry; _entries.Add(new GlusterFsEntry { - Name = BrickPath(entry.Name), + Name = BrickPath(relativePath), Size = entry.Size, IsDirectory = entry.IsDirectory, - DataFactory = entry.IsDirectory ? null : () => _extReader.Extract(captured), + DataFactory = entry.IsDirectory ? null : () => _extReader!.Extract(captured), }); } } + private void SetBrickRoot(IEnumerable paths) { + var roots = paths + .Select(NormalizePath) + .Where(IsGlusterIndexPath) + .Select(path => path.Equals(GlusterIndexName, StringComparison.Ordinal) + ? "" + : path[..^(GlusterIndexName.Length + 1)]) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + if (roots.Length > 1) + throw new NotSupportedException( + "GlusterFS: backing image contains multiple .glusterfs indexes; supply an image containing one brick."); + + this.HasGlusterIndex = roots.Length == 1; + this.BrickRoot = roots.FirstOrDefault() ?? ""; + } + + private bool TryGetBrickRelativePath(string path, out string relativePath) { + var normalized = NormalizePath(path); + if (this.BrickRoot.Length != 0) { + if (normalized.Equals(this.BrickRoot, StringComparison.Ordinal)) { + relativePath = ""; + return false; + } + + var prefix = this.BrickRoot + "/"; + if (!normalized.StartsWith(prefix, StringComparison.Ordinal)) { + relativePath = ""; + return false; + } + normalized = normalized[prefix.Length..]; + } + + if (normalized.Length == 0 || IsGlusterInternal(normalized)) { + relativePath = ""; + return false; + } + + relativePath = normalized; + return true; + } + private void AddMetadata(int backingEntryCount) { var metadata = BuildMetadata(backingEntryCount); _entries.Add(new GlusterFsEntry { @@ -156,6 +216,8 @@ private byte[] BuildMetadata(int backingEntryCount) { builder.Append(CultureInfo.InvariantCulture, $"backing_fs={this.BackingFileSystem}\n"); builder.Append(CultureInfo.InvariantCulture, $"image_size={_image.Length}\n"); builder.Append(CultureInfo.InvariantCulture, $"backing_entry_count={backingEntryCount}\n"); + builder.Append(CultureInfo.InvariantCulture, $"brick_root={this.BrickRoot}\n"); + builder.Append(CultureInfo.InvariantCulture, $"gluster_index_detected={this.HasGlusterIndex.ToString().ToLowerInvariant()}\n"); builder.Append("view=physical single-brick namespace\n"); builder.Append("gluster_internal_directory=.glusterfs (hidden from normal listing)\n"); builder.Append("xattrs=preserved in image but not interpreted\n"); @@ -165,16 +227,17 @@ private byte[] BuildMetadata(int backingEntryCount) { return Encoding.UTF8.GetBytes(builder.ToString()); } - private static string BrickPath(string path) { - var normalized = path.Replace('\\', '/').TrimStart('/'); - return "brick/" + normalized; - } + private static string BrickPath(string path) => "brick/" + NormalizePath(path); - private static bool IsGlusterInternal(string path) { - var normalized = path.Replace('\\', '/').Trim('/'); - return normalized.Equals(".glusterfs", StringComparison.Ordinal) || - normalized.StartsWith(".glusterfs/", StringComparison.Ordinal); - } + private static string NormalizePath(string path) => path.Replace('\\', '/').Trim('/'); + + private static bool IsGlusterIndexPath(string path) => + path.Equals(GlusterIndexName, StringComparison.Ordinal) || + path.EndsWith("/" + GlusterIndexName, StringComparison.Ordinal); + + private static bool IsGlusterInternal(string path) => + path.Equals(GlusterIndexName, StringComparison.Ordinal) || + path.StartsWith(GlusterIndexName + "/", StringComparison.Ordinal); private void ReadAt(long offset, Span destination) { var original = _image.Position; From d78fee33e98e669e0cebe743cace938060658ce2 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:08:28 +0200 Subject: [PATCH 082/165] Cover nested GlusterFS brick root discovery --- .../GlusterFs/GlusterFsDetectionTests.cs | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs b/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs index f670dae2a..ef0ff52e2 100644 --- a/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs +++ b/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs @@ -9,8 +9,9 @@ public class GlusterFsDetectionTests { private static byte[] BuildXfsBrick() { var writer = new FileSystem.Xfs.XfsWriter(); - writer.AddFile("data/hello.txt", "hello from xfs brick"u8.ToArray()); - writer.AddFile(".glusterfs/ab/internal-gfid", "backend index"u8.ToArray()); + writer.AddFile("host/brick/data/hello.txt", "hello from xfs brick"u8.ToArray()); + writer.AddFile("host/brick/.glusterfs/ab/internal-gfid", "backend index"u8.ToArray()); + writer.AddFile("outside.txt", "not part of the brick"u8.ToArray()); using var stream = new MemoryStream(); writer.WriteTo(stream); return stream.ToArray(); @@ -18,8 +19,9 @@ private static byte[] BuildXfsBrick() { private static byte[] BuildExtBrick() { var writer = new FileSystem.Ext.ExtWriter(); - writer.AddFile("data/hello.txt", "hello from ext brick"u8.ToArray()); - writer.AddFile(".glusterfs/cd/internal-gfid", "backend index"u8.ToArray()); + writer.AddFile("srv/brick/data/hello.txt", "hello from ext brick"u8.ToArray()); + writer.AddFile("srv/brick/.glusterfs/cd/internal-gfid", "backend index"u8.ToArray()); + writer.AddFile("outside.txt", "not part of the brick"u8.ToArray()); return writer.Build(); } @@ -41,7 +43,7 @@ public void Descriptor_UsesExtensionOnlyDetectionAndAdvertisesReadOnlyBrickView( } [Test, Category("HappyPath")] - public void XfsBackingStore_ListsBrickFilesAndHidesGlusterInternalIndex() { + public void XfsBackingStore_InfersBrickRootAndHidesNonBrickContentAndGlusterIndex() { var descriptor = new GlusterFsFormatDescriptor(); using var stream = new MemoryStream(BuildXfsBrick()); @@ -51,11 +53,12 @@ public void XfsBackingStore_ListsBrickFilesAndHidesGlusterInternalIndex() { Assert.That(names, Does.Contain("metadata.ini")); Assert.That(names, Does.Contain("brick/data/hello.txt")); Assert.That(names.Any(name => name.Contains(".glusterfs", StringComparison.Ordinal)), Is.False); + Assert.That(names, Does.Not.Contain("brick/outside.txt")); }); } [Test, Category("HappyPath")] - public void ExtBackingStore_ListsAndExtractsBrickFile() { + public void ExtBackingStore_InfersBrickRootAndExtractsBrickFile() { using var stream = new MemoryStream(BuildExtBrick()); using var reader = new GlusterFsReader(stream); @@ -64,8 +67,11 @@ public void ExtBackingStore_ListsAndExtractsBrickFile() { Assert.Multiple(() => { Assert.That(reader.ValidHeader, Is.True); Assert.That(reader.BackingFileSystem, Is.EqualTo("ext")); + Assert.That(reader.BrickRoot, Is.EqualTo("srv/brick")); + Assert.That(reader.HasGlusterIndex, Is.True); Assert.That(Encoding.UTF8.GetString(reader.Extract(entry)), Is.EqualTo("hello from ext brick")); Assert.That(reader.Entries.Any(candidate => candidate.Name.Contains(".glusterfs", StringComparison.Ordinal)), Is.False); + Assert.That(reader.Entries.Any(candidate => candidate.Name == "brick/outside.txt"), Is.False); }); } @@ -110,7 +116,7 @@ public void Descriptor_DoesNotAdvertiseMutationsThatWouldDiscardGlusterXattrs() } [Test, Category("HappyPath")] - public void Metadata_DescribesPhysicalViewAndMutationBlocker() { + public void Metadata_DescribesPhysicalViewBrickRootAndMutationBlocker() { using var stream = new MemoryStream(BuildXfsBrick()); using var reader = new GlusterFsReader(stream); var metadata = reader.Entries.Single(entry => entry.Name == "metadata.ini"); @@ -119,6 +125,8 @@ public void Metadata_DescribesPhysicalViewAndMutationBlocker() { Assert.Multiple(() => { Assert.That(text, Does.Contain("parse_status=single-brick-read-only")); Assert.That(text, Does.Contain("backing_fs=xfs")); + Assert.That(text, Does.Contain("brick_root=host/brick")); + Assert.That(text, Does.Contain("gluster_index_detected=true")); Assert.That(text, Does.Contain("view=physical single-brick namespace")); Assert.That(text, Does.Contain("xattrs=preserved in image but not interpreted")); Assert.That(text, Does.Contain("cluster_namespace_reconstruction=false")); From caec3b1f0b248ee085b16565944ccb4a827ac65a Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:08:49 +0200 Subject: [PATCH 083/165] + cover conservative Lustre maintenance paths --- .../Lustre/LustreDetectionTests.cs | 155 +++++++++++++++++- 1 file changed, 149 insertions(+), 6 deletions(-) diff --git a/Compression.Tests/Lustre/LustreDetectionTests.cs b/Compression.Tests/Lustre/LustreDetectionTests.cs index 7910ce7c2..3f5d9975d 100644 --- a/Compression.Tests/Lustre/LustreDetectionTests.cs +++ b/Compression.Tests/Lustre/LustreDetectionTests.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Text; using Compression.Registry; using FileSystem.Ext; @@ -27,13 +28,21 @@ public void Descriptor_IdentifiesByMagicAndExtension() { Assert.That(d.MagicSignatures, Has.Count.EqualTo(2)); Assert.That(d.MagicSignatures[0].Bytes, Is.EqualTo("LUSTRE"u8.ToArray())); Assert.That(d.MagicSignatures[1].Bytes, Is.EqualTo(new byte[] { 0x4C, 0x55, 0x73, 0x74 })); - // Promoted R/O — exposes multi-entry + directory capabilities; still no Create/Modify. + + // The namespace projection remains read-only: Lustre LMA/LOV/FID xattrs are opaque, + // so CanModify would over-promise add/remove/purge semantics. Maintenance below that + // layer is still safe where it only touches proven-free blocks / trailing geometry. Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanList), Is.True); Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanExtract), Is.True); Assert.That(d.Capabilities.HasFlag(FormatCapabilities.SupportsDirectories), Is.True); Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); Assert.That(d, Is.Not.InstanceOf()); + Assert.That(d, Is.Not.InstanceOf()); + Assert.That(d, Is.InstanceOf()); + Assert.That(d, Is.InstanceOf()); + Assert.That(d, Is.InstanceOf()); + Assert.That(d, Is.Not.InstanceOf()); } [Test, Category("HappyPath")] @@ -65,30 +74,82 @@ public void NoMagic_NoExt4_Throws() { Assert.Throws(() => d.List(ms, password: null)); } - [Test, Category("Stub")] + [Test, Category("ExceptionalCase")] + public void Legacy_Dump_MaintenanceRefusesNonLdiskfsBytes() { + var d = new LustreFormatDescriptor(); + using var wipe = new MemoryStream(BuildMinimalLegacy(), writable: true); + Assert.Throws(() => ((IWipeEmpty)d).WipeUnusedSpace(wipe)); + + using var source = new MemoryStream(BuildMinimalLegacy(), writable: false); + using var target = new MemoryStream(); + Assert.Throws(() => ((IArchiveShrinkable)d).Shrink(source, target)); + } + + [Test, Category("HappyPath")] public void Description_FlagsLdiskfsDelegationAndOutOfScope() { var d = new LustreFormatDescriptor(); var desc = d.Description.ToLowerInvariant(); Assert.That(desc, Does.Contain("ldiskfs")); Assert.That(desc, Does.Contain("ext4")); Assert.That(desc, Does.Contain("out of scope")); + Assert.That(desc, Does.Contain("shrink")); + Assert.That(desc, Does.Contain("wipe")); } - // ── ldiskfs (ext4) R/O delegation path ────────────────────────────────── + // ── ldiskfs (ext4) delegation + conservative maintenance ──────────────── + + private static byte[] BuildLdiskfsImage(string volumeLabel, params (string Name, byte[] Data)[] files) + => BuildLdiskfsImage(4096, volumeLabel, files); - private static byte[] BuildLdiskfsImage(string volumeLabel, params (string Name, byte[] Data)[] files) { + private static byte[] BuildLdiskfsImage(int totalBlocks, string volumeLabel, params (string Name, byte[] Data)[] files) { var writer = new ExtWriter(); foreach (var (name, data) in files) writer.AddFile(name, data); // ext4 + journal, 4 KB blocks, volume label set (Lustre convention: "lustre-OST0000" / "MGS" / "lustre-MDT0000"). return writer.Build( blockSize: 4096, - totalBlocks: 4096, + totalBlocks: totalBlocks, version: ExtWriter.ExtVersion.Ext4, journal: true, volumeLabel: volumeLabel, inodeSize: 256); } + private static (int BlockSize, long BlockOffset, long DescriptorOffset) FindInitializedFreeBlock(byte[] image) { + var sb = image.AsSpan(1024, 1024); + var blockSize = 1024 << (int)BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(24, 4)); + var firstDataBlock = BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(20, 4)); + var blocksPerGroup = BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(32, 4)); + var featureIncompat = BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(96, 4)); + ulong totalBlocks = BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(4, 4)); + if ((featureIncompat & 0x80) != 0) + totalBlocks |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(0x150, 4)) << 32; + var descriptorSize = (featureIncompat & 0x80) != 0 + ? Math.Max(32, BinaryPrimitives.ReadUInt16LittleEndian(sb.Slice(0xFE, 2))) + : 32; + var groupCount = (totalBlocks - firstDataBlock + blocksPerGroup - 1) / blocksPerGroup; + var descriptorTableOffset = (long)(firstDataBlock + 1UL) * blockSize; + + for (ulong group = 0; group < groupCount; ++group) { + var descriptorOffset = descriptorTableOffset + (long)group * descriptorSize; + var descriptor = image.AsSpan((int)descriptorOffset, descriptorSize); + var flags = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.Slice(0x12, 2)); + if ((flags & 0x0002) != 0) continue; // BLOCK_UNINIT + + ulong bitmapBlock = BinaryPrimitives.ReadUInt32LittleEndian(descriptor[..4]); + if (descriptorSize >= 64) + bitmapBlock |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(descriptor.Slice(0x20, 4)) << 32; + var bitmapOffset = checked((long)bitmapBlock * blockSize); + var bitmap = image.AsSpan((int)bitmapOffset, blockSize); + var groupFirst = firstDataBlock + group * blocksPerGroup; + var blocksInGroup = (int)Math.Min((ulong)blocksPerGroup, totalBlocks - groupFirst); + for (var bit = 0; bit < blocksInGroup; ++bit) + if ((bitmap[bit >> 3] & (1 << (bit & 7))) == 0) + return (blockSize, checked((long)(groupFirst + (uint)bit) * blockSize), descriptorOffset); + } + + throw new AssertionException("Test image unexpectedly contains no initialized free ldiskfs block."); + } + [Test, Category("HappyPath")] public void Ldiskfs_Image_RecognisedAndDelegated() { var d = new LustreFormatDescriptor(); @@ -108,7 +169,7 @@ public void Ldiskfs_Image_RecognisedAndDelegated() { } [Test, Category("HappyPath")] - public void Ldiskfs_Metadata_DocumentsPartialRoStatus() { + public void Ldiskfs_Metadata_DocumentsPartialBackingStoreStatus() { var d = new LustreFormatDescriptor(); var data = BuildLdiskfsImage("lustre-MDT0000", ("CONFIGS/mountdata", "lustre MDT config"u8.ToArray())); @@ -120,6 +181,7 @@ public void Ldiskfs_Metadata_DocumentsPartialRoStatus() { Assert.That(text, Does.Contain("parse_status=partial-ldiskfs")); Assert.That(text, Does.Contain("backing_fs=ldiskfs (ext4-compatible)")); Assert.That(text, Does.Contain("ldiskfs_volume_label=lustre-MDT0000")); + Assert.That(text, Does.Contain("bitmap-proven free blocks")); Assert.That(text, Does.Contain("not the Lustre logical view").IgnoreCase .Or.Contain("NOT the Lustre logical view")); } @@ -146,6 +208,87 @@ public void Ldiskfs_Extract_ReturnsFileBytesFromDelegatedReader() { "ldiskfs delegation must return the exact file bytes the ext4 reader produces."); } + [Test, Category("HappyPath")] + public void Ldiskfs_Wipe_ZeroesBitmapFreeBlocks_WithoutTouchingLiveObjects() { + var d = new LustreFormatDescriptor(); + var payload = Enumerable.Range(0, 1536).Select(i => (byte)(i * 17)).ToArray(); + var data = BuildLdiskfsImage("lustre-OST0000", ("OBJECTS/0_42", payload)); + var (blockSize, freeOffset, _) = FindInitializedFreeBlock(data); + data.AsSpan((int)freeOffset, blockSize).Fill(0xA5); + + using var image = new MemoryStream(data, writable: true); + var wiped = ((IWipeEmpty)d).WipeUnusedSpace(image, wipeClusterTips: true, wipeDeletedEntries: true); + + Assert.That(wiped, Is.GreaterThanOrEqualTo(blockSize)); + var after = image.ToArray(); + Assert.That(after.AsSpan((int)freeOffset, blockSize).ToArray(), Is.All.EqualTo((byte)0)); + + using var verify = new MemoryStream(after, writable: false); + using var entry = ((IArchiveFormatOperations)d).OpenEntry(verify, "ldiskfs/OBJECTS/0_42", password: null); + using var actual = new MemoryStream(); + entry.CopyTo(actual); + Assert.That(actual.ToArray(), Is.EqualTo(payload), + "Bitmap-driven wipe must not change any allocated object bytes."); + } + + [Test, Category("BoundaryCase")] + public void Ldiskfs_Wipe_FailsClosedForBlockUninitGroup() { + var d = new LustreFormatDescriptor(); + var data = BuildLdiskfsImage("lustre-OST0000", ("OBJECTS/0_1", "live"u8.ToArray())); + var (blockSize, freeOffset, descriptorOffset) = FindInitializedFreeBlock(data); + data.AsSpan((int)freeOffset, blockSize).Fill(0x5A); + + var flags = BinaryPrimitives.ReadUInt16LittleEndian(data.AsSpan((int)descriptorOffset + 0x12, 2)); + BinaryPrimitives.WriteUInt16LittleEndian(data.AsSpan((int)descriptorOffset + 0x12, 2), (ushort)(flags | 0x0002)); + + using var image = new MemoryStream(data, writable: true); + _ = ((IWipeEmpty)d).WipeUnusedSpace(image); + + Assert.That(image.ToArray().AsSpan((int)freeOffset, blockSize).ToArray(), Is.All.EqualTo((byte)0x5A), + "BLOCK_UNINIT means the on-disk bitmap is not authoritative; the whole group must be left alone."); + } + + [Test, Category("HappyPath")] + public void Ldiskfs_Shrink_TrimsTrailingFreeBlocks_AndPreservesTargetIdentityAndPayload() { + var d = new LustreFormatDescriptor(); + var payload = Enumerable.Range(0, 8192).Select(i => (byte)(i ^ (i >> 8))).ToArray(); + var data = BuildLdiskfsImage(8192, "lustre-MDT0000", ("OBJECTS/0_7", payload)); + + using var source = new MemoryStream(data, writable: false); + using var target = new MemoryStream(); + ((IArchiveShrinkable)d).Shrink(source, target); + + Assert.That(target.Length, Is.LessThan(data.LongLength), + "A small target in an oversized one-group ldiskfs image should lose its trailing free blocks."); + + target.Position = 0; + using var meta = ((IArchiveFormatOperations)d).OpenEntry(target, "metadata.ini", password: null); + using var sr = new StreamReader(meta); + Assert.That(sr.ReadToEnd(), Does.Contain("ldiskfs_volume_label=lustre-MDT0000"), + "Shrink must retain the Lustre target label rather than rebuilding as generic ext."); + + target.Position = 0; + using var entry = ((IArchiveFormatOperations)d).OpenEntry(target, "ldiskfs/OBJECTS/0_7", password: null); + using var actual = new MemoryStream(); + entry.CopyTo(actual); + Assert.That(actual.ToArray(), Is.EqualTo(payload)); + } + + [Test, Category("HappyPath")] + public void Ldiskfs_LayoutAnalysis_ReportsGeometryWithoutClaimingStructuralRelayout() { + var d = new LustreFormatDescriptor(); + var data = BuildLdiskfsImage("lustre-OST0000", ("OBJECTS/0_1", "payload"u8.ToArray())); + using var image = new MemoryStream(data, writable: false); + + var analysis = ((ILayoutOptimizable)d).AnalyzeLayout(image); + + Assert.That(analysis.CurrentUnitSize, Is.EqualTo(4096)); + Assert.That(analysis.OptimalUnitSize, Is.EqualTo(4096)); + Assert.That(analysis.PotentialSavingsBytes, Is.Zero); + Assert.That(analysis.RequiresRebuild, Is.Empty); + Assert.That(analysis.Notes.Any(n => n.Contains("not offered", StringComparison.OrdinalIgnoreCase)), Is.True); + } + [Test, Category("BoundaryCase")] public void Ldiskfs_Reader_PreservesRawImageAsLustreObjectBin() { var d = new LustreFormatDescriptor(); From 2f0555978699ac2e23abe74584a5fac5db1bbaef Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:08:51 +0200 Subject: [PATCH 084/165] Fail closed on unsupported TUX3 block sizes --- FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs b/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs index 9aac1144f..51749513f 100644 --- a/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs +++ b/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs @@ -155,7 +155,9 @@ public void Shrink(Stream input, Stream output) { private static bool TryGetDeclaredVolumeLength(Tux3Reader reader, out long length) { length = 0; - if (!reader.ValidSuperblock || reader.VolBlocks == 0 || reader.BlockBits >= 63) + // The reference format reserves a 4 KiB superblock area as the maximum block size. + // Treat larger shifts as corrupt rather than trusting them for destructive maintenance. + if (!reader.ValidSuperblock || reader.VolBlocks == 0 || reader.BlockBits > 12) return false; var blockSize = 1UL << reader.BlockBits; From 7c3c072025b792bd200f22c13c139d90d3d6b10b Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:09:33 +0200 Subject: [PATCH 085/165] Implement eCryptfs lower-file codec --- .../FileSystem.Ecryptfs/EcryptfsCodec.cs | 555 ++++++++++++++++++ 1 file changed, 555 insertions(+) create mode 100644 FileSystems/FileSystem.Ecryptfs/EcryptfsCodec.cs diff --git a/FileSystems/FileSystem.Ecryptfs/EcryptfsCodec.cs b/FileSystems/FileSystem.Ecryptfs/EcryptfsCodec.cs new file mode 100644 index 000000000..d7d0150db --- /dev/null +++ b/FileSystems/FileSystem.Ecryptfs/EcryptfsCodec.cs @@ -0,0 +1,555 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace FileSystem.Ecryptfs; + +internal static class EcryptfsCodec { + internal const uint MarkerMagic = 0x3C81B7F5; + internal const int FixedHeaderSize = 26; + internal const int DefaultExtentSize = 4096; + internal const int MinimumMetadataSize = 8192; + internal const byte CurrentFileVersion = 3; + internal const uint EncryptedFlag = 0x00000002; + internal const uint MetadataInXattrFlag = 0x00000004; + internal const int PassphraseHashIterations = 65536; + + private const byte Tag3 = 0x8C; + private const byte Tag11 = 0xED; + private const byte Tag1 = 0x01; + private const byte Aes128 = 0x07; + private const byte Aes192 = 0x08; + private const byte Aes256 = 0x09; + + private static ReadOnlySpan DefaultSalt => [0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77]; + private static ReadOnlySpan ConsoleName => "_CONSOLE"u8; + + internal sealed record PassphrasePacket( + byte CipherCode, + byte[] Salt, + byte[] EncryptedFileKey, + byte[] Signature + ) { + public int KeySize => CipherCode switch { + Aes128 => 16, + Aes192 => 24, + Aes256 => 32, + _ => 0, + }; + + public string CipherDescription => CipherCode switch { + Aes128 => "AES-128-CBC", + Aes192 => "AES-192-CBC", + Aes256 => "AES-256-CBC", + _ => $"cipher-0x{CipherCode:X2}", + }; + } + + internal sealed record Header( + ulong DecryptedSize, + uint MarkerWord, + uint Flags, + uint ExtentSize, + ushort HeaderExtentCount, + int MetadataSize, + int PacketEndOffset, + bool PaddingIsKnown, + IReadOnlyList PassphrasePackets + ) { + public byte FileVersion => (byte)(Flags >> 24); + + public long CanonicalLength { + get { + var extents = DecryptedSize == 0 ? 0UL : checked((DecryptedSize + ExtentSize - 1) / ExtentSize); + var dataBytes = checked(extents * ExtentSize); + var total = checked((ulong)MetadataSize + dataBytes); + return total <= long.MaxValue + ? (long)total + : throw new InvalidDataException("eCryptfs lower-file length exceeds the supported stream range."); + } + } + + public string CipherDescription + => PassphrasePackets.Count == 0 ? "encrypted" : PassphrasePackets[0].CipherDescription; + } + + internal static Header ReadHeader(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead || !stream.CanSeek) + throw new ArgumentException("eCryptfs parsing requires a readable, seekable stream.", nameof(stream)); + + var originalPosition = stream.Position; + try { + Span fixedHeader = stackalloc byte[FixedHeaderSize]; + stream.Position = 0; + ReadExactly(stream, fixedHeader); + + var extentSize = BinaryPrimitives.ReadUInt32BigEndian(fixedHeader[20..24]); + var headerExtentCount = BinaryPrimitives.ReadUInt16BigEndian(fixedHeader[24..26]); + ValidateGeometry(extentSize, headerExtentCount); + + var metadataSize64 = checked((ulong)extentSize * headerExtentCount); + if (metadataSize64 > int.MaxValue) + throw new InvalidDataException("eCryptfs metadata region is too large to parse."); + var metadataSize = (int)metadataSize64; + if (stream.Length < metadataSize) + throw new InvalidDataException("Truncated eCryptfs metadata region."); + + var metadata = new byte[metadataSize]; + stream.Position = 0; + ReadExactly(stream, metadata); + var header = ParseHeader(metadata); + if (stream.Length < header.CanonicalLength) + throw new InvalidDataException("Truncated eCryptfs ciphertext extents."); + return header; + } finally { + stream.Position = originalPosition; + } + } + + internal static void DecryptTo(Stream input, Stream output, Header header, string password) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(password); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("eCryptfs decryption requires a readable, seekable input.", nameof(input)); + if (!output.CanWrite) + throw new ArgumentException("eCryptfs decryption requires a writable output.", nameof(output)); + + var packet = FindMatchingPassphrasePacket(header, password, out var fekek); + try { + if (packet.KeySize == 0) + throw new NotSupportedException($"eCryptfs cipher code 0x{packet.CipherCode:X2} is not supported; AES-128/192/256 are supported."); + + var fileKey = DecryptFileKey(packet, fekek); + try { + var rootIv = MD5.HashData(fileKey); + var encrypted = new byte[header.ExtentSize]; + var remaining = header.DecryptedSize; + var extentIndex = 0L; + input.Position = header.MetadataSize; + + using var aes = Aes.Create(); + aes.Mode = CipherMode.CBC; + aes.Padding = PaddingMode.None; + aes.Key = fileKey; + + while (remaining > 0) { + ReadExactly(input, encrypted); + var iv = DeriveExtentIv(rootIv, extentIndex++); + using var transform = aes.CreateDecryptor(aes.Key, iv); + var plaintext = transform.TransformFinalBlock(encrypted, 0, encrypted.Length); + var toWrite = (int)Math.Min((ulong)plaintext.Length, remaining); + output.Write(plaintext, 0, toWrite); + remaining -= (ulong)toWrite; + CryptographicOperations.ZeroMemory(plaintext); + } + } finally { + CryptographicOperations.ZeroMemory(fileKey); + } + } finally { + CryptographicOperations.ZeroMemory(fekek); + } + } + + internal static void Create(Stream output, ReadOnlySpan plaintext, string password, int keySize = 16) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(password); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("eCryptfs creation requires a writable, seekable output.", nameof(output)); + if (keySize is not (16 or 24 or 32)) + throw new ArgumentOutOfRangeException(nameof(keySize), "AES key size must be 16, 24, or 32 bytes."); + + var passwordBytes = Encoding.UTF8.GetBytes(password); + if (passwordBytes.Length > 64) + throw new ArgumentException("eCryptfs passphrases are limited to 64 encoded bytes.", nameof(password)); + + var metadata = new byte[MinimumMetadataSize]; + BinaryPrimitives.WriteUInt64BigEndian(metadata.AsSpan(0, 8), (ulong)plaintext.Length); + + Span markerBytes = stackalloc byte[4]; + RandomNumberGenerator.Fill(markerBytes); + var markerWord = BinaryPrimitives.ReadUInt32BigEndian(markerBytes); + BinaryPrimitives.WriteUInt32BigEndian(metadata.AsSpan(8, 4), markerWord); + BinaryPrimitives.WriteUInt32BigEndian(metadata.AsSpan(12, 4), markerWord ^ MarkerMagic); + BinaryPrimitives.WriteUInt32BigEndian(metadata.AsSpan(16, 4), ((uint)CurrentFileVersion << 24) | EncryptedFlag); + BinaryPrimitives.WriteUInt32BigEndian(metadata.AsSpan(20, 4), DefaultExtentSize); + BinaryPrimitives.WriteUInt16BigEndian(metadata.AsSpan(24, 2), MinimumMetadataSize / DefaultExtentSize); + + var fileKey = RandomNumberGenerator.GetBytes(keySize); + var (fekek, signature) = DerivePassphraseMaterial(passwordBytes, DefaultSalt); + try { + var encryptedFileKey = EncryptFileKey(fileKey, fekek, keySize); + try { + var offset = FixedHeaderSize; + offset = WriteTag3(metadata, offset, keySize, encryptedFileKey, DefaultSalt); + offset = WriteTag11(metadata, offset, signature); + + output.Position = 0; + output.SetLength(0); + output.Write(metadata); + + if (!plaintext.IsEmpty) { + var rootIv = MD5.HashData(fileKey); + using var aes = Aes.Create(); + aes.Mode = CipherMode.CBC; + aes.Padding = PaddingMode.None; + aes.Key = fileKey; + + var plainExtent = new byte[DefaultExtentSize]; + var consumed = 0; + var extentIndex = 0L; + while (consumed < plaintext.Length) { + Array.Clear(plainExtent); + var count = Math.Min(DefaultExtentSize, plaintext.Length - consumed); + plaintext.Slice(consumed, count).CopyTo(plainExtent); + var iv = DeriveExtentIv(rootIv, extentIndex++); + using var transform = aes.CreateEncryptor(aes.Key, iv); + var encrypted = transform.TransformFinalBlock(plainExtent, 0, plainExtent.Length); + output.Write(encrypted); + CryptographicOperations.ZeroMemory(encrypted); + consumed += count; + } + CryptographicOperations.ZeroMemory(plainExtent); + } + output.Position = 0; + } finally { + CryptographicOperations.ZeroMemory(encryptedFileKey); + } + } finally { + CryptographicOperations.ZeroMemory(fileKey); + CryptographicOperations.ZeroMemory(fekek); + CryptographicOperations.ZeroMemory(signature); + CryptographicOperations.ZeroMemory(passwordBytes); + } + } + + internal static int ResolveKeySize(string? encryptionMethod) { + if (string.IsNullOrWhiteSpace(encryptionMethod)) return 16; + return encryptionMethod.Trim().ToLowerInvariant() switch { + "aes" or "aes128" or "aes-128" or "aes-128-cbc" => 16, + "aes192" or "aes-192" or "aes-192-cbc" => 24, + "aes256" or "aes-256" or "aes-256-cbc" => 32, + _ => throw new NotSupportedException($"Unsupported eCryptfs encryption method '{encryptionMethod}'. Use AES-128, AES-192, or AES-256."), + }; + } + + internal static long WipeUnusedSpace(Stream image) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("eCryptfs wipe requires a readable, writable, seekable stream.", nameof(image)); + + var header = ReadHeader(image); + long wiped = 0; + if (header.PaddingIsKnown && header.PacketEndOffset < header.MetadataSize) + wiped += ZeroRange(image, header.PacketEndOffset, header.MetadataSize - header.PacketEndOffset); + if (image.Length > header.CanonicalLength) + wiped += ZeroRange(image, header.CanonicalLength, image.Length - header.CanonicalLength); + image.Position = 0; + return wiped; + } + + internal static void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("eCryptfs shrink requires a readable, seekable input.", nameof(input)); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("eCryptfs shrink requires a writable, seekable output.", nameof(output)); + + var header = ReadHeader(input); + input.Position = 0; + output.Position = 0; + output.SetLength(0); + CopyExactly(input, output, header.CanonicalLength); + output.Position = 0; + } + + internal static void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("eCryptfs purge requires a readable, writable, seekable stream.", nameof(archive)); + + var header = ReadHeader(archive); + Span zeroSize = stackalloc byte[8]; + archive.Position = 0; + archive.Write(zeroSize); + archive.SetLength(header.MetadataSize); + archive.Position = 0; + } + + private static Header ParseHeader(ReadOnlySpan metadata) { + if (metadata.Length < FixedHeaderSize) + throw new InvalidDataException("Truncated eCryptfs fixed header."); + + var decryptedSize = BinaryPrimitives.ReadUInt64BigEndian(metadata[..8]); + var markerWord = BinaryPrimitives.ReadUInt32BigEndian(metadata[8..12]); + var markerCheck = BinaryPrimitives.ReadUInt32BigEndian(metadata[12..16]); + if ((markerWord ^ MarkerMagic) != markerCheck) + throw new InvalidDataException("Invalid eCryptfs marker pair."); + + var flags = BinaryPrimitives.ReadUInt32BigEndian(metadata[16..20]); + var version = (byte)(flags >> 24); + if (version is 0 or > CurrentFileVersion) + throw new InvalidDataException($"Unsupported eCryptfs file version {version}."); + if ((flags & EncryptedFlag) == 0) + throw new InvalidDataException("The eCryptfs header does not mark the lower file as encrypted."); + if ((flags & MetadataInXattrFlag) != 0) + throw new InvalidDataException("This standalone eCryptfs stream declares xattr metadata; the required xattr is not part of the stream."); + + var extentSize = BinaryPrimitives.ReadUInt32BigEndian(metadata[20..24]); + var headerExtentCount = BinaryPrimitives.ReadUInt16BigEndian(metadata[24..26]); + ValidateGeometry(extentSize, headerExtentCount); + var metadataSize64 = checked((ulong)extentSize * headerExtentCount); + if (metadataSize64 > (ulong)metadata.Length) + throw new InvalidDataException("Truncated eCryptfs metadata region."); + var metadataSize = (int)metadataSize64; + + var packets = new List(); + var offset = FixedHeaderSize; + var paddingKnown = true; + while (offset < metadataSize) { + var tag = metadata[offset]; + if (tag == 0) break; + + if (tag == Tag3) { + var tag3 = ParsePacket(metadata[..metadataSize], ref offset, Tag3); + var packet = ParseTag3(tag3); + if (offset >= metadataSize || metadata[offset] != Tag11) + throw new InvalidDataException("eCryptfs tag-3 packet is not followed by its tag-11 signature packet."); + var tag11 = ParsePacket(metadata[..metadataSize], ref offset, Tag11); + packet = packet with { Signature = ParseTag11Signature(tag11) }; + packets.Add(packet); + continue; + } + + if (tag == Tag1) { + _ = ParsePacket(metadata[..metadataSize], ref offset, Tag1); + continue; + } + + // Unknown non-zero metadata is potentially live. Stop parsing and refuse to + // call the remainder free space; this keeps wipe fail-closed. + paddingKnown = false; + offset = metadataSize; + break; + } + + return new Header(decryptedSize, markerWord, flags, extentSize, headerExtentCount, + metadataSize, offset, paddingKnown, packets); + } + + private static PassphrasePacket ParseTag3(ReadOnlySpan body) { + if (body.Length < 14) + throw new InvalidDataException("Truncated eCryptfs tag-3 packet."); + if (body[0] != 0x04) + throw new InvalidDataException($"Unsupported eCryptfs tag-3 version {body[0]}."); + if (body[2] != 0x03) + throw new InvalidDataException($"Unsupported eCryptfs S2K specifier {body[2]}."); + if (body[3] != 0x01) + throw new InvalidDataException($"Unsupported eCryptfs tag-3 hash identifier {body[3]}."); + if (body[12] != 0x60) + throw new InvalidDataException($"Unsupported eCryptfs S2K iteration code 0x{body[12]:X2}."); + + var encryptedKey = body[13..].ToArray(); + if (encryptedKey.Length == 0 || encryptedKey.Length > 64 || (encryptedKey.Length & 15) != 0) + throw new InvalidDataException("Invalid eCryptfs encrypted file-key length."); + return new PassphrasePacket(body[1], body[4..12].ToArray(), encryptedKey, []); + } + + private static byte[] ParseTag11Signature(ReadOnlySpan body) { + if (body.Length < 14 || body[0] != 0x62) + throw new InvalidDataException("Invalid eCryptfs tag-11 literal packet."); + var nameLength = body[1]; + var signatureOffset = 2 + nameLength + 4; + if (signatureOffset > body.Length || body.Length - signatureOffset != 8) + throw new InvalidDataException("Invalid eCryptfs tag-11 signature length."); + return body[signatureOffset..].ToArray(); + } + + private static ReadOnlySpan ParsePacket(ReadOnlySpan data, ref int offset, byte expectedTag) { + if (offset >= data.Length || data[offset++] != expectedTag) + throw new InvalidDataException($"Expected eCryptfs packet tag 0x{expectedTag:X2}."); + var bodyLength = ReadPacketLength(data, ref offset); + if (bodyLength < 0 || bodyLength > data.Length - offset) + throw new InvalidDataException("Truncated eCryptfs packet body."); + var body = data.Slice(offset, bodyLength); + offset += bodyLength; + return body; + } + + private static int ReadPacketLength(ReadOnlySpan data, ref int offset) { + if (offset >= data.Length) + throw new InvalidDataException("Truncated eCryptfs packet length."); + var first = data[offset++]; + if (first < 192) return first; + if (first < 224) { + if (offset >= data.Length) + throw new InvalidDataException("Truncated two-byte eCryptfs packet length."); + return ((first - 192) << 8) + data[offset++] + 192; + } + throw new InvalidDataException("Unsupported eCryptfs partial/five-byte packet length."); + } + + private static int WriteTag3(Span metadata, int offset, int keySize, ReadOnlySpan encryptedFileKey, ReadOnlySpan salt) { + var bodyLength = 13 + encryptedFileKey.Length; + metadata[offset++] = Tag3; + metadata[offset++] = checked((byte)bodyLength); + metadata[offset++] = 0x04; + metadata[offset++] = keySize switch { 16 => Aes128, 24 => Aes192, 32 => Aes256, _ => throw new ArgumentOutOfRangeException(nameof(keySize)) }; + metadata[offset++] = 0x03; + metadata[offset++] = 0x01; + salt.CopyTo(metadata[offset..]); + offset += salt.Length; + metadata[offset++] = 0x60; + encryptedFileKey.CopyTo(metadata[offset..]); + return offset + encryptedFileKey.Length; + } + + private static int WriteTag11(Span metadata, int offset, ReadOnlySpan signature) { + const int bodyLength = 22; + metadata[offset++] = Tag11; + metadata[offset++] = bodyLength; + metadata[offset++] = 0x62; + metadata[offset++] = 8; + ConsoleName.CopyTo(metadata[offset..]); + offset += ConsoleName.Length; + metadata.Slice(offset, 4).Clear(); + offset += 4; + signature.CopyTo(metadata[offset..]); + return offset + signature.Length; + } + + private static PassphrasePacket FindMatchingPassphrasePacket(Header header, string password, out byte[] fekek) { + var passwordBytes = Encoding.UTF8.GetBytes(password); + if (passwordBytes.Length > 64) + throw new CryptographicException("eCryptfs passphrase exceeds 64 encoded bytes."); + try { + foreach (var packet in header.PassphrasePackets) { + var material = DerivePassphraseMaterial(passwordBytes, packet.Salt); + if (CryptographicOperations.FixedTimeEquals(material.Signature, packet.Signature)) { + CryptographicOperations.ZeroMemory(material.Signature); + fekek = material.Fekek; + return packet; + } + CryptographicOperations.ZeroMemory(material.Fekek); + CryptographicOperations.ZeroMemory(material.Signature); + } + } finally { + CryptographicOperations.ZeroMemory(passwordBytes); + } + if (header.PassphrasePackets.Count == 0) + throw new NotSupportedException("This eCryptfs file has no passphrase tag-3 packet; private-key authentication is not supported."); + throw new CryptographicException("The supplied eCryptfs passphrase does not match any authentication-token signature."); + } + + private static (byte[] Fekek, byte[] Signature) DerivePassphraseMaterial(ReadOnlySpan password, ReadOnlySpan salt) { + var seed = new byte[salt.Length + password.Length]; + salt.CopyTo(seed); + password.CopyTo(seed.AsSpan(salt.Length)); + + var digest = SHA512.HashData(seed); + CryptographicOperations.ZeroMemory(seed); + for (var i = 1; i < PassphraseHashIterations; ++i) { + var next = SHA512.HashData(digest); + CryptographicOperations.ZeroMemory(digest); + digest = next; + } + + var sigHash = SHA512.HashData(digest); + var signature = sigHash.AsSpan(0, 8).ToArray(); + CryptographicOperations.ZeroMemory(sigHash); + return (digest, signature); + } + + private static byte[] EncryptFileKey(ReadOnlySpan fileKey, ReadOnlySpan fekek, int keySize) { + var padded = new byte[keySize == 24 ? 32 : keySize]; + fileKey.CopyTo(padded); + try { + using var aes = Aes.Create(); + aes.Mode = CipherMode.ECB; + aes.Padding = PaddingMode.None; + aes.Key = fekek[..keySize].ToArray(); + using var transform = aes.CreateEncryptor(); + return transform.TransformFinalBlock(padded, 0, padded.Length); + } finally { + CryptographicOperations.ZeroMemory(padded); + } + } + + private static byte[] DecryptFileKey(PassphrasePacket packet, ReadOnlySpan fekek) { + if (packet.KeySize == 0) + throw new NotSupportedException($"Unsupported eCryptfs cipher code 0x{packet.CipherCode:X2}."); + using var aes = Aes.Create(); + aes.Mode = CipherMode.ECB; + aes.Padding = PaddingMode.None; + aes.Key = fekek[..packet.KeySize].ToArray(); + using var transform = aes.CreateDecryptor(); + var decrypted = transform.TransformFinalBlock(packet.EncryptedFileKey, 0, packet.EncryptedFileKey.Length); + if (decrypted.Length < packet.KeySize) { + CryptographicOperations.ZeroMemory(decrypted); + throw new InvalidDataException("eCryptfs encrypted file key is shorter than the selected cipher key size."); + } + var fileKey = decrypted.AsSpan(0, packet.KeySize).ToArray(); + CryptographicOperations.ZeroMemory(decrypted); + return fileKey; + } + + private static byte[] DeriveExtentIv(ReadOnlySpan rootIv, long extentIndex) { + Span source = stackalloc byte[32]; + source.Clear(); + rootIv[..16].CopyTo(source); + var text = extentIndex.ToString(CultureInfo.InvariantCulture); + if (!Encoding.ASCII.TryGetBytes(text, source[16..31], out _)) + throw new InvalidDataException("eCryptfs extent index is too large for IV derivation."); + return MD5.HashData(source); + } + + private static void ValidateGeometry(uint extentSize, ushort headerExtentCount) { + if (extentSize < 16 || (extentSize & (extentSize - 1)) != 0 || (extentSize & 15) != 0) + throw new InvalidDataException($"Invalid eCryptfs extent size {extentSize}."); + if (headerExtentCount == 0) + throw new InvalidDataException("eCryptfs header extent count is zero."); + var metadataSize = checked((ulong)extentSize * headerExtentCount); + if (metadataSize < FixedHeaderSize) + throw new InvalidDataException("eCryptfs metadata region is smaller than the fixed header."); + } + + private static long ZeroRange(Stream stream, long offset, long count) { + if (count <= 0) return 0; + var zeroes = new byte[64 * 1024]; + stream.Position = offset; + var remaining = count; + while (remaining > 0) { + var chunk = (int)Math.Min(zeroes.Length, remaining); + stream.Write(zeroes, 0, chunk); + remaining -= chunk; + } + return count; + } + + private static void CopyExactly(Stream input, Stream output, long count) { + var buffer = new byte[64 * 1024]; + var remaining = count; + while (remaining > 0) { + var requested = (int)Math.Min(buffer.Length, remaining); + var read = input.Read(buffer, 0, requested); + if (read <= 0) throw new EndOfStreamException(); + output.Write(buffer, 0, read); + remaining -= read; + } + } + + private static void ReadExactly(Stream stream, Span buffer) { + var total = 0; + while (total < buffer.Length) { + var read = stream.Read(buffer[total..]); + if (read <= 0) throw new EndOfStreamException("Unexpected end of eCryptfs stream."); + total += read; + } + } + + private static void ReadExactly(Stream stream, byte[] buffer) => ReadExactly(stream, buffer.AsSpan()); +} From 64cccb2c8872a8e17b1efe14233f180414c506fd Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:09:46 +0200 Subject: [PATCH 086/165] Restore unchanged README support badge --- Hawkynt.FileFormats.FileSystems/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index 063577775..69ddc46be 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -88,7 +88,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [DriveSpace 3 CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace3` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | `dmsdos` mounts the genuine `GenuineDvr3Writer` output and reads byte-exact | Win95 DVR3 CVF; a Win95 guest oracle and the HiPack codec are still open | [DriveSpace 3 notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/FileSystems/FileSystem.DriveSpace3/FORMAT-NOTES.md) | | [DriveSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` driver under QEMU mounts, reads and writes; real driver CVFs read byte-exact | MSDSP6.0 CVF; stored clusters | [DriveSpace](https://en.wikipedia.org/wiki/DriveSpace) | | [exFAT](https://en.wikipedia.org/wiki/ExFAT) | `ExFat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `exfat` mount r/w + [`fsck.exfat`](https://github.com/exfatprogs/exfatprogs) | — | [exFAT specification](https://learn.microsoft.com/windows/win32/fileio/exfat-specification) | -| [FAT Filesystem Image](https://en.wikipedia.org/wiki/File_Allocation_Table) | `Fat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount r/w + [`fsck.fat`](https://github.com/dosfstools/dosfstools) + [`mtools`](https://www.gnu.org/software/mtools/) + reverse [`mkfs.vfat`](https://github.com/dosfstools/dosfstools) | FAT12/16/32 with long filenames; in-place `FatModifier` | [FAT specification](https://download.microsoft.com/download/1/6/1/161ba512-40e2-4cc9-843a-923143f3456c/fatgen103.doc) | +| [FAT Filesystem Image](https://en.wikipedia.org/wiki/File_Allocation_Table) | `Fat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount r/w + [`fsck.fat`](https://github.com/dosfstools/dosfstools) + [`mtools`](https://www.gnu.org/software/mtools) + reverse [`mkfs.vfat`](https://github.com/dosfstools/dosfstools) | FAT12/16/32 with long filenames; in-place `FatModifier` | [FAT specification](https://download.microsoft.com/download/1/6/1/161ba512-40e2-4cc9-843a-923143f3456c/fatgen103.doc) | | [FAT+ Filesystem Image (large-file extension)](https://en.wikipedia.org/wiki/File_Allocation_Table#FAT+) | `FatPlus` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount + [`fsck.fat`](https://github.com/dosfstools/dosfstools) read the written volume | FAT+ large-file extension | [FAT+ draft](https://www.fdos.org/kernel/fatplus.txt) | | [FATX (Xbox)](https://en.wikipedia.org/wiki/FATX) | `Fatx` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Xbox FATX | [FATX (xboxdevwiki)](https://xboxdevwiki.net/FATX) | | [HPFS](https://en.wikipedia.org/wiki/High_Performance_File_System) | `Hpfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | OS/2 HPFS | [HPFS layout (OS/2 Museum)](https://www.os2museum.com/wp/the-hpfs-disk-layout/) | @@ -123,7 +123,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Minix V2 FS](https://en.wikipedia.org/wiki/MINIX_file_system) | `MinixV2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.minix`](https://github.com/util-linux/util-linux) clean after in-place edit + host-kernel loop-mount reads the added file byte-exact | 30-character names | [MINIX filesystem](https://en.wikipedia.org/wiki/MINIX_file_system) | | [NILFS v1](https://en.wikipedia.org/wiki/NILFS) | `Nilfs1` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | NILFS v1 | [NILFS](https://nilfs.sourceforge.io/) | | [NILFS2](https://en.wikipedia.org/wiki/NILFS) | `Nilfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel loop-mount reads the in-place-added file byte-exact; [`mkfs.nilfs2`](https://nilfs.sourceforge.io/) superblock re-validated | — | [NILFS](https://nilfs.sourceforge.io/) | -| [OCFS2 (Oracle Cluster Filesystem 2)](https://en.wikipedia.org/wiki/OCFS2) | `Ocfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.ocfs2 -fn`](https://github.com/markfasheh/ocfs2-tools) + `debugfs.ocfs2`; `mkfs.ocfs2` image read by the package | Edits rebuild the volume | [OCFS2](https://www.kernel.org/doc/html/latest/filesystems/ocfs2.html) | +| [OCFS2 (Oracle Cluster Filesystem 2)](https://en.wikipedia.org/wiki/OCFS2) | `Ocfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.ocfs2 -fn`](https://github.com/markfasheh/ocfs2-tools) + `debugfs.ocfs2`; `mkfs.ocfs2` image read by the package | Edits rebuild the volume | [OCFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | | [QNX4 FS](https://en.wikipedia.org/wiki/QNX4FS) | `Qnx4` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx4](https://www.kernel.org/doc/html/latest/filesystems/qnx4.html) | | [QNX6 Neutrino FS](https://en.wikipedia.org/wiki/QNX6FS) | `Qnx6` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx6](https://www.kernel.org/doc/html/latest/filesystems/qnx6.html) | | [Reiser4](https://en.wikipedia.org/wiki/Reiser4) | `Reiser4` | WORM | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; `reiser4progs` not installable here | Empty tree from `mkfs.reiser4` captures; files live in a payload area the reiser4 driver does not see, so the writer stays WORM | [Reiser4](https://reiser4.wiki.kernel.org/) | @@ -134,7 +134,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [UNIX System V FS](https://en.wikipedia.org/wiki/Unix_File_System) | `SysV` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `sysv` mount reads byte-exact | System V filesystem | [sysv](https://www.kernel.org/doc/html/latest/filesystems/sysv-fs.html) | | [VxFS (Veritas)](https://en.wikipedia.org/wiki/Veritas_File_System) | `VxFs` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | Linux `freevxfs` driver mounts the written volume | One fileset, direct extents, flat root; edits rebuild the volume | [freevxfs](https://github.com/torvalds/linux/tree/master/fs/freevxfs) | | [Xenix FS](https://en.wikipedia.org/wiki/Xenix) | `Xenix` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `sysv` (`detect_xenix`) mount reads byte-exact | — | [sysv](https://www.kernel.org/doc/html/latest/filesystems/sysv-fs.html) | -| [XFS](https://en.wikipedia.org/wiki/XFS) | `Xfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `xfs` mount r/w + [`xfs_repair -n`](https://git.kernel.org/pub/scm/fs/xfs/xfsprogs-dev.git) | XFS v5 | [XFS documentation](https://kernel.org/doc/html/latest/filesystems/xfs/index.html) | +| [XFS](https://en.wikipedia.org/wiki/XFS) | `Xfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `xfs` mount r/w + [`xfs_repair -n`](https://git.kernel.org/pub/scm/fs/xfsprogs-dev.git) | XFS v5 | [XFS documentation](https://kernel.org/doc/html/latest/filesystems/xfs/index.html) | | [ZFS](https://en.wikipedia.org/wiki/ZFS) | `Zfs` | R/W | ✅ | ✅ moving | — | ✅ | ✅ | ✅ | [`zdb -l`](https://github.com/openzfs/zfs) label path; own reader + struct-parity tests | OpenZFS v28 single-vdev profile; edits rebuild the pool | [OpenZFS documentation](https://openzfs.github.io/openzfs-docs/) | ### Apple @@ -344,7 +344,7 @@ cwb recover raw.img --mode files --format Jpeg,Png If this project saves you time or money, consider supporting its development: [![GitHub Sponsors](https://img.shields.io/badge/GitHub-Sponsor-EA4AAA?logo=githubsponsors)](https://github.com/sponsors/Hawkynt) -[![PayPal](https://img.shields.io/badge/PayPal-Donate-00457C)](https://www.paypal.me/hawkynt) +[![PayPal](https://img.shields.io/badge/PayPal-Donate-00457C?logo=paypal)](https://www.paypal.me/hawkynt) ## 📜 License From 008e6e71fc18aaccca0f79a6664781bbf7437974 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:09:47 +0200 Subject: [PATCH 087/165] + robust UEFI firmware volume R/W and maintenance Implement PI-spec-aware FFS2/FFS3 parsing and writing, transactional offline edits, erase-polarity handling, signed-volume mutation refusal, native purge, erase-aware wipe/layout mapping, and conservative offline defragmentation that preserves fixed/aligned/VTF records. Add regression and round-trip coverage for FFS3 large files, failed replacement atomicity, defrag, wipe, purge, and erase-polarity-zero state encoding. --- .../UefiFv/UefiFvMaintenanceTests.cs | 129 ++++++++++++++++++ .../FileFormat.UefiFv/UefiFvConstants.cs | 59 ++++++++ .../UefiFvFormatDescriptor.cs | 93 +++++-------- .../UefiFvInPlaceModifier.cs | 129 +++++++----------- .../FileFormat.UefiFv/UefiFvLayoutMap.cs | 59 ++++++++ .../FileFormat.UefiFv/UefiFvMaintenance.cs | 104 ++++++++++++++ FileFormats/FileFormat.UefiFv/UefiFvParser.cs | 105 ++++++++++++++ FileFormats/FileFormat.UefiFv/UefiFvReader.cs | 114 ++++------------ FileFormats/FileFormat.UefiFv/UefiFvWriter.cs | 81 ++++++----- 9 files changed, 604 insertions(+), 269 deletions(-) create mode 100644 Compression.Tests/UefiFv/UefiFvMaintenanceTests.cs create mode 100644 FileFormats/FileFormat.UefiFv/UefiFvConstants.cs create mode 100644 FileFormats/FileFormat.UefiFv/UefiFvLayoutMap.cs create mode 100644 FileFormats/FileFormat.UefiFv/UefiFvMaintenance.cs create mode 100644 FileFormats/FileFormat.UefiFv/UefiFvParser.cs diff --git a/Compression.Tests/UefiFv/UefiFvMaintenanceTests.cs b/Compression.Tests/UefiFv/UefiFvMaintenanceTests.cs new file mode 100644 index 000000000..fdcc524d4 --- /dev/null +++ b/Compression.Tests/UefiFv/UefiFvMaintenanceTests.cs @@ -0,0 +1,129 @@ +using System.Buffers.Binary; +using Compression.Registry; +using FileFormat.UefiFv; + +namespace Compression.Tests.UefiFv; + +[TestFixture] +public sealed class UefiFvMaintenanceTests { + private const string DriverName = "11223344-5566-7788-99aa-bbccddeeff00_DRIVER.bin"; + private const string RawName = "01234567-89ab-cdef-0123-456789abcdef_RAW.bin"; + + [Test, Category("RoundTrip")] + public void Create_LargeFile_UsesFfs3Header2() { + var payload = Enumerable.Range(0, 0x1000000).Select(i => (byte)(i * 17)).ToArray(); + var descriptor = new UefiFvFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, [ArchiveInputInfo.InMemory(DriverName, payload)], new FormatCreateOptions()); + + var bytes = image.ToArray(); + var fv = UefiFvReader.Read(bytes); + Assert.Multiple(() => { + Assert.That(fv.Header.FileSystemGuid, Is.EqualTo(Guid.Parse("5473C07A-3DCB-4DCA-BD6F-1E9689E7349A"))); + Assert.That(fv.Files, Has.Count.EqualTo(1)); + Assert.That(fv.Files[0].Attributes & 0x01, Is.EqualTo(0x01)); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(72 + 20, 4)) & 0x00FFFFFF, Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(bytes.AsSpan(72 + 24, 8)), Is.EqualTo((ulong)payload.Length + 32)); + Assert.That(fv.Files[0].Contents, Is.EqualTo(payload)); + }); + } + + [Test, Category("Regression")] + public void Replace_WhenNoRunFits_IsTransactional() { + var descriptor = new UefiFvFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, new byte[1024])], new FormatCreateOptions()); + ((IArchiveModifiable)descriptor).Add(image, [ArchiveInputInfo.InMemory(RawName, new byte[60000])]); + var before = image.ToArray(); + + Assert.That(() => ((IArchiveModifiable)descriptor).Add(image, + [ArchiveInputInfo.InMemory(DriverName, new byte[12000])]), Throws.InstanceOf()); + Assert.That(image.ToArray(), Is.EqualTo(before), "failed replacement must not erase the previous file"); + } + + [Test, Category("RoundTrip")] + public void Defrag_ClosesErasedHole_AndPreservesPayload() { + var descriptor = new UefiFvFormatDescriptor(); + var first = Enumerable.Range(0, 257).Select(i => (byte)i).ToArray(); + var second = Enumerable.Range(0, 721).Select(i => (byte)(i * 7)).ToArray(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, first), ArchiveInputInfo.InMemory(RawName, second)], new FormatCreateOptions()); + ((IArchiveModifiable)descriptor).Remove(image, [DriverName]); + + ((IArchiveDefragmentable)descriptor).Defragment(image); + + image.Position = 0; + var layout = ((IArchiveLayoutMap)descriptor).EnumerateLayout(image).ToList(); + var used = layout.Single(e => e.Kind == DefragBlockKind.Used); + Assert.That(used.Offset, Is.EqualTo(72)); + AssertFiles(image, (RawName, second)); + } + + [Test, Category("RoundTrip")] + public void Wipe_UsesErasePolarity_AndClearsDeletedRecord() { + var descriptor = new UefiFvFormatDescriptor(); + var deleted = Enumerable.Repeat((byte)0xA5, 257).ToArray(); + var live = Enumerable.Repeat((byte)0x5A, 333).ToArray(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, deleted), ArchiveInputInfo.InMemory(RawName, live)], new FormatCreateOptions()); + + var bytes = image.ToArray(); + bytes[72 + 23] = 0xE8; // erase-polarity-one encoding of EFI_FILE_DELETED reached after DATA_VALID + image.Position = 0; + image.Write(bytes); + image.Position = 0; + + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(image, wipeDeletedEntries: true); + Assert.That(wiped, Is.GreaterThan(0)); + var after = image.ToArray(); + var deletedFootprint = (24 + deleted.Length + 7) & ~7; + Assert.That(after.AsSpan(72, deletedFootprint).ToArray(), Is.All.EqualTo((byte)0xFF)); + AssertFiles(image, (RawName, live)); + } + + [Test, Category("RoundTrip")] + public void Purge_LeavesValidEmptyVolume() { + var descriptor = new UefiFvFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, new byte[512]), ArchiveInputInfo.InMemory(RawName, new byte[128])], + new FormatCreateOptions()); + + ((IArchivePurgeable)descriptor).Purge(image); + + var fv = UefiFvReader.Read(image.ToArray()); + Assert.That(fv.Files.Where(f => f.Type != 0xF0), Is.Empty); + image.Position = 0; + Assert.That(descriptor.List(image, null).Select(e => e.Name), Is.EqualTo(new[] { "metadata.ini" })); + } + + [Test, Category("EdgeCase")] + public void Reader_HonorsErasePolarityZeroStateEncoding() { + var descriptor = new UefiFvFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, new byte[] { 1, 2, 3, 4 })], new FormatCreateOptions()); + var bytes = image.ToArray(); + var attributes = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(44)) & ~0x00000800u; + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(44), attributes); + bytes[72 + 23] = 0x07; + bytes.AsSpan((72 + 24 + 4 + 7) & ~7).Clear(); + + var fv = UefiFvReader.Read(bytes); + Assert.Multiple(() => { + Assert.That(fv.Header.ErasePolarity, Is.False); + Assert.That(fv.Files, Has.Count.EqualTo(1)); + Assert.That(fv.Files[0].Contents, Is.EqualTo(new byte[] { 1, 2, 3, 4 })); + }); + } + + private static void AssertFiles(MemoryStream image, params (string Name, byte[] Data)[] expected) { + var actual = UefiFvReader.Read(image.ToArray()).Files.Where(f => f.Type != 0xF0) + .ToDictionary(f => $"{f.Name:D}_{UefiFvReader.ShortTypeTag(f.Type)}.bin", f => f.Contents, StringComparer.OrdinalIgnoreCase); + Assert.That(actual.Keys, Is.EquivalentTo(expected.Select(e => e.Name))); + foreach (var (name, data) in expected) Assert.That(actual[name], Is.EqualTo(data), name); + } +} diff --git a/FileFormats/FileFormat.UefiFv/UefiFvConstants.cs b/FileFormats/FileFormat.UefiFv/UefiFvConstants.cs new file mode 100644 index 000000000..ee95b432e --- /dev/null +++ b/FileFormats/FileFormat.UefiFv/UefiFvConstants.cs @@ -0,0 +1,59 @@ +#pragma warning disable CS1591 + +namespace FileFormat.UefiFv; + +internal static class UefiFvConstants { + internal const uint ErasePolarityMask = 0x00000800; + internal const byte LargeFile = 0x01; + internal const byte Alignment2 = 0x02; + internal const byte Fixed = 0x04; + internal const byte Alignment = 0x38; + internal const byte HeaderConstruction = 0x01; + internal const byte HeaderValid = 0x02; + internal const byte DataValid = 0x04; + internal const byte MarkedForUpdate = 0x08; + internal const byte Deleted = 0x10; + internal const byte HeaderInvalid = 0x20; + internal static readonly Guid Ffs2Guid = Guid.Parse("8C8CE578-8A3D-4F1C-9935-896185C32DD3"); + internal static readonly Guid Ffs3Guid = Guid.Parse("5473C07A-3DCB-4DCA-BD6F-1E9689E7349A"); + internal static readonly Guid VolumeTopGuid = Guid.Parse("1BA0062E-C779-4582-8566-336AE8F78F09"); + internal static readonly Guid SignedContentsGuid = Guid.Parse("0F9D89E8-9259-4F76-A5AF-0C89E34023DF"); + + internal static byte EffectiveState(byte raw, bool erasePolarity) { + var value = (byte)((erasePolarity ? ~raw : raw) & 0x3F); + if ((value & HeaderInvalid) != 0) return HeaderInvalid; + if ((value & Deleted) != 0) return Deleted; + if ((value & MarkedForUpdate) != 0) return MarkedForUpdate; + if ((value & DataValid) != 0) return DataValid; + if ((value & HeaderValid) != 0) return HeaderValid; + return (value & HeaderConstruction) != 0 ? HeaderConstruction : (byte)0; + } + + internal static byte EncodeState(byte cumulativeState, bool erasePolarity) + => erasePolarity ? unchecked((byte)~cumulativeState) : cumulativeState; + + internal static int DataAlignment(byte attributes) { + var code = (attributes & Alignment) >> 3; + if ((attributes & Alignment2) != 0) return 1 << (17 + code); + return code switch { 0 => 1, 1 => 16, 2 => 128, 3 => 512, 4 => 1024, 5 => 4096, 6 => 32768, _ => 65536 }; + } + + internal static int Align8(int value) => checked((int)(((long)value + 7) & ~7L)); + internal static int AlignOffset(int start, int absolute) => checked(start + Align8(checked(absolute - start))); +} + +internal sealed record UefiFvSlot(int Offset, int HeaderSize, int Size, int Footprint, Guid Name, + byte Type, byte Attributes, byte RawState, byte State) { + public int DataOffset => checked(this.Offset + this.HeaderSize); + public int DataLength => checked(this.Size - this.HeaderSize); + public int DataEnd => checked(this.Offset + this.Size); + public int End => checked(this.Offset + this.Footprint); + public bool IsPad => this.Type == 0xF0; + public bool IsFixed => (this.Attributes & UefiFvConstants.Fixed) != 0; + public int DataAlignment => UefiFvConstants.DataAlignment(this.Attributes); +} + +internal sealed record UefiFvLayout(int Start, int End, int DataStart, Guid FileSystemGuid, uint Attributes, + bool ErasePolarity, byte EraseByte, int? UsedSizeOffset, bool IsSigned, IReadOnlyList Slots) { + public bool SupportsLargeFiles => this.FileSystemGuid == UefiFvConstants.Ffs3Guid; +} diff --git a/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs b/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs index f23aae820..260247d73 100644 --- a/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs +++ b/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs @@ -7,87 +7,41 @@ namespace FileFormat.UefiFv; /// -/// UEFI PI Firmware Volume (.fv/.fd) archive surface. FFS files are -/// exposed as {GUID}_{TYPE_TAG}.bin; standalone volumes can be created and -/// ordinary FFS2 records can be added/replaced/removed through erased free space. +/// UEFI PI Firmware Volume archive surface. Standard FFS2/FFS3 files are exposed as +/// {GUID}_{TYPE_TAG}.bin; mutable unsigned volumes support transactional edits, +/// erase-aware wiping, purge, and offline consolidation. /// -/// References: -/// -/// https://uefi.org/specifications — UEFI Platform Initialization (PI) Specification, Volume 3: Firmware Storage Design -/// https://github.com/LongSoft/UEFITool — UEFITool firmware-volume parser/editor -/// +/// References: UEFI PI Specification 1.10 Volume III (Firmware Storage) and EDK II +/// as an interoperability oracle. The implementation is independently written. /// public sealed class UefiFvFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, - IArchiveCreatable, IArchiveModifiable { + IArchiveCreatable, IArchiveModifiable, IArchiveDefragmentable, IArchiveLayoutMap, IArchivePurgeable { - /// - /// Gets the id. - /// public string Id => "UefiFv"; - /// - /// Gets the display name. - /// public string DisplayName => "UEFI Firmware Volume"; - /// - /// Gets the category. - /// public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; - /// - /// Gets the default extension. - /// public string DefaultExtension => ".fv"; - /// - /// Gets the extensions. - /// public IReadOnlyList Extensions => [".fv", ".fd"]; - /// - /// Gets the compound extensions. - /// public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// public IReadOnlyList MagicSignatures => [ - new([(byte)'_', (byte)'F', (byte)'V', (byte)'H'], - Offset: UefiFvReader.SignatureOffset, Confidence: 0.95), + new([(byte)'_', (byte)'F', (byte)'V', (byte)'H'], Offset: UefiFvReader.SignatureOffset, Confidence: 0.95), ]; - /// - /// Gets the methods. - /// public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// public string Description => - "UEFI PI Firmware Volume — create and offline R/W for ordinary FFS2 files in fixed-capacity volumes."; + "UEFI PI Firmware Volume — FFS2/FFS3 create and transactional offline R/W, purge, wipe and defrag."; - /// - /// Lists the entries in the supplied container. - /// public List List(Stream stream, string? password) => BuildEntries(stream).Select((e, i) => new ArchiveEntryInfo( Index: i, Name: e.Name, OriginalSize: e.Data.LongLength, CompressedSize: e.Data.LongLength, Method: e.Method, IsDirectory: false, IsEncrypted: false, LastModified: null)).ToList(); - /// - /// Decodes the supplied input. - /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { foreach (var e in BuildEntries(stream)) { if (files != null && files.Length > 0 && !MatchesFilter(e.Name, files)) continue; @@ -109,6 +63,25 @@ public void Remove(Stream archive, string[] entryNames) => UefiFvInPlaceModifier.Remove(archive, entryNames.Where(n => !string.Equals(n, "metadata.ini", StringComparison.OrdinalIgnoreCase)).ToArray()); + public void Defragment(Stream archive) + => UefiFvMaintenance.Defragment(archive, new DefragOptions { Mode = DefragMode.ConsolidateAtStart }); + + public void Defragment(Stream archive, DefragOptions options) + => UefiFvMaintenance.Defragment(archive, options); + + public IEnumerable EnumerateLayout(Stream archive) + => UefiFvLayoutMap.Enumerate(archive); + + /// + /// Restores unused bytes to the FV's declared erase value. For erase-polarity-one + /// flash this is 0xFF rather than zero; writing zero would turn free space into + /// apparently programmed data and make the firmware volume structurally invalid. + /// + public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) + => UefiFvLayoutMap.Wipe(image, wipeDeletedEntries); + + public void Purge(Stream archive) => UefiFvMaintenance.Purge(archive); + private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) { if (stream.CanSeek) stream.Position = 0; using var ms = new MemoryStream(); @@ -117,9 +90,7 @@ public void Remove(Stream archive, string[] entryNames) var fvStart = UefiFvReader.FindFirst(data) ?? 0; var fv = UefiFvReader.Read(data, fvStart); - var entries = new List<(string, byte[], string)> { - ("metadata.ini", BuildMetadata(fv), "stored"), - }; + var entries = new List<(string, byte[], string)> { ("metadata.ini", BuildMetadata(fv), "stored") }; foreach (var f in fv.Files) { if (f.Type == 0xF0) continue; entries.Add((UefiFvWriter.EntryName(f.Name, f.Type), f.Contents, "stored")); @@ -134,20 +105,18 @@ private static byte[] BuildMetadata(UefiFvReader.FirmwareVolume fv) { sb.Append(CultureInfo.InvariantCulture, $"file_system_guid = {fv.Header.FileSystemGuid:D}\n"); sb.Append(CultureInfo.InvariantCulture, $"fv_length = {fv.Header.FvLength}\n"); sb.Append(CultureInfo.InvariantCulture, $"attributes = 0x{fv.Header.Attributes:X8}\n"); + sb.Append(CultureInfo.InvariantCulture, $"erase_byte = 0x{fv.Header.EraseByte:X2}\n"); sb.Append(CultureInfo.InvariantCulture, $"header_length = {fv.Header.HeaderLength}\n"); sb.Append(CultureInfo.InvariantCulture, $"checksum = 0x{fv.Header.Checksum:X4}\n"); sb.Append(CultureInfo.InvariantCulture, $"ext_header_offset = 0x{fv.Header.ExtHeaderOffset:X}\n"); sb.Append(CultureInfo.InvariantCulture, $"revision = {fv.Header.Revision}\n"); sb.Append(CultureInfo.InvariantCulture, $"file_count = {fv.Files.Count}\n"); - sb.AppendLine(); sb.AppendLine("[block_map]"); for (var i = 0; i < fv.Header.BlockMap.Count; i++) { - var (nb, bl) = fv.Header.BlockMap[i]; - sb.Append(CultureInfo.InvariantCulture, - $"block_{i} = {nb} blocks x {bl} bytes\n"); + var (blocks, length) = fv.Header.BlockMap[i]; + sb.Append(CultureInfo.InvariantCulture, $"block_{i} = {blocks} blocks x {length} bytes\n"); } - sb.AppendLine(); sb.AppendLine("[files]"); for (var i = 0; i < fv.Files.Count; i++) { diff --git a/FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs b/FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs index f9f0d6b8f..53c84bdc8 100644 --- a/FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs +++ b/FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs @@ -4,46 +4,42 @@ namespace FileFormat.UefiFv; -/// -/// Offline random-access editor for ordinary FFS2 records in a firmware volume. -/// It reuses erased 0xFF ranges and never relocates unrelated FFS files. -/// +/// Transactional offline editor for standard PI FFS2/FFS3 firmware volumes. internal static class UefiFvInPlaceModifier { - private sealed record Slot(int Offset, int Length, Guid Guid, byte Type) { - public string Name => UefiFvWriter.EntryName(Guid, Type); - } - public static void Add(Stream archive, IReadOnlyList inputs) { ArgumentNullException.ThrowIfNull(inputs); var state = Open(archive); foreach (var (name, data) in FilesOnly(inputs)) { var identity = UefiFvWriter.IdentityFromName(name); - var existing = ScanSlots(state.Image, state.FvStart, state.FvEnd) - .FirstOrDefault(s => s.Guid == identity.Guid); - if (existing != null) - Erase(state, existing.Offset, existing.Length); + foreach (var old in UefiFvParser.LiveSlots(state.Volume, includePad: false) + .Where(slot => slot.Name == identity.Guid).ToList()) + Erase(state, old.Offset, old.Footprint); + Refresh(state); - var encoded = UefiFvWriter.BuildFfsFile(identity.Guid, identity.Type, data); + var encoded = UefiFvWriter.BuildFfsFile(identity.Guid, identity.Type, data, + state.Volume.FileSystemGuid, state.Volume.ErasePolarity); var footprint = UefiFvWriter.Align8(encoded.Length); - var offset = FindErasedRun(state.Image, state.DataStart, state.FvEnd, footprint); + var offset = FindErasedRun(state.Image, state.Volume.DataStart, state.Volume.End, + footprint, state.Volume.EraseByte); if (offset < 0) throw new IOException($"UEFI FV has no erased run large enough for '{name}' ({footprint} bytes)."); - - Write(state, offset, encoded); - if (footprint > encoded.Length) - Erase(state, offset + encoded.Length, footprint - encoded.Length); + encoded.CopyTo(state.Image, offset); + Refresh(state); } + Finish(state); } public static void Remove(Stream archive, string[] entryNames) { ArgumentNullException.ThrowIfNull(entryNames); var state = Open(archive); foreach (var name in entryNames) { - var slot = ScanSlots(state.Image, state.FvStart, state.FvEnd) - .FirstOrDefault(s => string.Equals(s.Name, name, StringComparison.OrdinalIgnoreCase)); - if (slot != null) - Erase(state, slot.Offset, slot.Length); + var slot = UefiFvParser.LiveSlots(state.Volume, includePad: false) + .FirstOrDefault(s => string.Equals(UefiFvWriter.EntryName(s.Name, s.Type), name, StringComparison.OrdinalIgnoreCase)); + if (slot == null) continue; + Erase(state, slot.Offset, slot.Footprint); + Refresh(state); } + Finish(state); } private static EditorState Open(Stream archive) { @@ -52,76 +48,47 @@ private static EditorState Open(Stream archive) { throw new ArgumentException("UEFI FV mutation requires a seekable read/write stream.", nameof(archive)); if (archive.Length > int.MaxValue) throw new NotSupportedException("The in-memory FV editor supports images up to 2 GiB."); - archive.Position = 0; var image = new byte[checked((int)archive.Length)]; archive.ReadExactly(image); - var fvStart = UefiFvReader.FindFirst(image) - ?? throw new InvalidDataException("UEFI FV header was not found."); - var fv = UefiFvReader.Read(image, fvStart); - var fvEnd = checked(fvStart + (int)fv.Header.FvLength); - if (fvEnd > image.Length) throw new InvalidDataException("UEFI FV extends past the image."); - var dataStart = Align8(checked(fvStart + fv.Header.HeaderLength)); - return new EditorState(archive, image, fvStart, dataStart, fvEnd); + var start = UefiFvReader.FindFirst(image) ?? throw new InvalidDataException("UEFI FV header was not found."); + var volume = UefiFvParser.Parse(image, start); + EnsureMutable(volume); + return new EditorState(archive, image, start, volume); } - private static List ScanSlots(byte[] image, int fvStart, int fvEnd) { - var fv = UefiFvReader.Read(image, fvStart); - var pos = Align8(checked(fvStart + fv.Header.HeaderLength)); - var result = new List(); - while (pos + UefiFvWriter.FfsHeaderLength <= fvEnd) { - // One quantum at a time, and only the quantum is tested — see the same - // walk in UefiFvReader: testing a whole header straddles the end of an - // erased gap and reads the next file's GUID as a length. - if (IsErased(image.AsSpan(pos, UefiFvWriter.Alignment))) { - pos += UefiFvWriter.Alignment; - continue; - } - - var size = image[pos + 20] | (image[pos + 21] << 8) | (image[pos + 22] << 16); - if (size < UefiFvWriter.FfsHeaderLength || pos + size > fvEnd) - throw new InvalidDataException($"Invalid FFS file header at FV offset 0x{pos - fvStart:X}."); - var guid = new Guid(image.AsSpan(pos, 16)); - var type = image[pos + 18]; - var footprint = UefiFvWriter.Align8(size); - result.Add(new Slot(pos, footprint, guid, type)); - pos += footprint; - } - return result; + private static void EnsureMutable(UefiFvLayout volume) { + if (volume.FileSystemGuid != UefiFvConstants.Ffs2Guid && volume.FileSystemGuid != UefiFvConstants.Ffs3Guid) + throw new NotSupportedException($"UEFI FV uses unsupported file-system GUID {volume.FileSystemGuid:D}."); + if (volume.IsSigned) + throw new NotSupportedException("Signed UEFI firmware volumes cannot be modified without re-signing the complete volume."); } - private static int FindErasedRun(byte[] image, int start, int end, int needed) { - for (var pos = Align8(start); pos + needed <= end; pos += UefiFvWriter.Alignment) { - if (IsErased(image.AsSpan(pos, needed))) return pos; - } - return -1; - } + private static void Refresh(EditorState state) + => state.Volume = UefiFvParser.Parse(state.Image, state.Start); - private static bool IsErased(ReadOnlySpan bytes) { - foreach (var b in bytes) - if (b != 0xFF) return false; - return true; + private static void Finish(EditorState state) { + var usedEnd = UefiFvParser.LiveSlots(state.Volume).Select(slot => slot.DataEnd) + .DefaultIfEmpty(state.Volume.DataStart).Max(); + UefiFvParser.WriteUsedSize(state.Image, state.Volume, usedEnd); + state.Archive.Position = 0; + state.Archive.Write(state.Image); + state.Archive.SetLength(state.Image.Length); } - private static void Write(EditorState state, int offset, ReadOnlySpan bytes) { - bytes.CopyTo(state.Image.AsSpan(offset, bytes.Length)); - state.Archive.Position = offset; - state.Archive.Write(bytes); + private static int FindErasedRun(byte[] image, int start, int end, int needed, byte eraseByte) { + for (var pos = UefiFvConstants.AlignOffset(start, start); pos + needed <= end; pos += UefiFvWriter.Alignment) + if (UefiFvParser.IsErased(image.AsSpan(pos, needed), eraseByte)) return pos; + return -1; } - private static void Erase(EditorState state, int offset, int length) { - state.Image.AsSpan(offset, length).Fill(0xFF); - state.Archive.Position = offset; - Span erased = stackalloc byte[1024]; - erased.Fill(0xFF); - var remaining = length; - while (remaining > 0) { - var take = Math.Min(remaining, erased.Length); - state.Archive.Write(erased[..take]); - remaining -= take; - } - } + private static void Erase(EditorState state, int offset, int length) + => state.Image.AsSpan(offset, length).Fill(state.Volume.EraseByte); - private static int Align8(int value) => (value + 7) & ~7; - private sealed record EditorState(Stream Archive, byte[] Image, int FvStart, int DataStart, int FvEnd); + private sealed class EditorState(Stream archive, byte[] image, int start, UefiFvLayout volume) { + public Stream Archive { get; } = archive; + public byte[] Image { get; } = image; + public int Start { get; } = start; + public UefiFvLayout Volume { get; set; } = volume; + } } diff --git a/FileFormats/FileFormat.UefiFv/UefiFvLayoutMap.cs b/FileFormats/FileFormat.UefiFv/UefiFvLayoutMap.cs new file mode 100644 index 000000000..2652ef314 --- /dev/null +++ b/FileFormats/FileFormat.UefiFv/UefiFvLayoutMap.cs @@ -0,0 +1,59 @@ +#pragma warning disable CS1591 +using Compression.Registry; + +namespace FileFormat.UefiFv; + +internal static class UefiFvLayoutMap { + internal static IEnumerable Enumerate(Stream archive) { + var (image, volume) = UefiFvMaintenance.Read(archive, writable: false); + return Build(image.Length, volume, includeDeletedAsFree: true); + } + + internal static long Wipe(Stream archive, bool wipeDeletedEntries) { + var (image, volume) = UefiFvMaintenance.Read(archive, writable: true); + UefiFvMaintenance.EnsureMutable(volume); + long written = 0; + foreach (var extent in Build(image.Length, volume, wipeDeletedEntries).Where(e => e.Kind == DefragBlockKind.Free)) { + image.AsSpan(checked((int)extent.Offset), checked((int)extent.Length)).Fill(volume.EraseByte); + written += extent.Length; + } + + var reparsed = UefiFvParser.Parse(image, volume.Start); + var usedEnd = UefiFvParser.LiveSlots(reparsed).Select(slot => slot.DataEnd).DefaultIfEmpty(reparsed.DataStart).Max(); + UefiFvParser.WriteUsedSize(image, reparsed, usedEnd); + UefiFvMaintenance.Commit(archive, image); + return written; + } + + private static List Build(int imageLength, UefiFvLayout volume, bool includeDeletedAsFree) { + var result = new List(); + if (volume.DataStart > 0) + result.Add(new DefragBlockInfo(0, volume.DataStart, DefragBlockKind.MetadataReserved, "FV header / prefix")); + + var cursor = volume.DataStart; + foreach (var slot in volume.Slots.OrderBy(slot => slot.Offset)) { + if (cursor < slot.Offset) + result.Add(new DefragBlockInfo(cursor, slot.Offset - cursor, DefragBlockKind.Free)); + + DefragBlockKind kind; + string? name = null; + if (UefiFvParser.IsLive(volume, slot)) { + kind = slot.IsPad ? DefragBlockKind.MetadataReserved : DefragBlockKind.Used; + name = slot.IsPad ? "FFS pad" : UefiFvWriter.EntryName(slot.Name, slot.Type); + } else if (includeDeletedAsFree && UefiFvParser.IsDiscardable(volume, slot)) { + kind = DefragBlockKind.Free; + } else { + kind = DefragBlockKind.MetadataReserved; + name = "Incomplete FFS record"; + } + result.Add(new DefragBlockInfo(slot.Offset, slot.Footprint, kind, name)); + cursor = slot.End; + } + + if (cursor < volume.End) + result.Add(new DefragBlockInfo(cursor, volume.End - cursor, DefragBlockKind.Free)); + if (volume.End < imageLength) + result.Add(new DefragBlockInfo(volume.End, imageLength - volume.End, DefragBlockKind.MetadataReserved, "FV suffix")); + return result; + } +} diff --git a/FileFormats/FileFormat.UefiFv/UefiFvMaintenance.cs b/FileFormats/FileFormat.UefiFv/UefiFvMaintenance.cs new file mode 100644 index 000000000..1ec81d8b6 --- /dev/null +++ b/FileFormats/FileFormat.UefiFv/UefiFvMaintenance.cs @@ -0,0 +1,104 @@ +#pragma warning disable CS1591 +using Compression.Registry; + +namespace FileFormat.UefiFv; + +internal static class UefiFvMaintenance { + internal static void Defragment(Stream archive, DefragOptions options) { + ArgumentNullException.ThrowIfNull(options); + if (options.Mode != DefragMode.ConsolidateAtStart) + throw new NotSupportedException($"UEFI FV defrag supports only {DefragMode.ConsolidateAtStart}."); + + var (image, volume) = OpenWritable(archive); + options.CancellationToken.ThrowIfCancellationRequested(); + options.OnProgress?.Invoke(new DefragProgressEvent("scanning", 0, -1, -1, image.Length, null, "Scanning UEFI firmware volume")); + + var output = image.ToArray(); + output.AsSpan(volume.DataStart, volume.End - volume.DataStart).Fill(volume.EraseByte); + var live = UefiFvParser.LiveSlots(volume).ToList(); + var anchored = live.Where(IsAnchored).ToList(); + foreach (var slot in anchored) + image.AsSpan(slot.Offset, slot.Size).CopyTo(output.AsSpan(slot.Offset, slot.Size)); + + var movable = live.Where(slot => !IsAnchored(slot)).ToList(); + for (var i = 0; i < movable.Count; i++) { + options.CancellationToken.ThrowIfCancellationRequested(); + var slot = movable[i]; + var footprint = UefiFvConstants.Align8(slot.Size); + var destination = FindRun(output, volume, footprint); + if (destination < 0) + throw new InvalidDataException($"UEFI FV cannot place {slot.Name:D} while preserving fixed/aligned files."); + image.AsSpan(slot.Offset, slot.Size).CopyTo(output.AsSpan(destination, slot.Size)); + options.OnProgress?.Invoke(new DefragProgressEvent("writing", (i + 1d) / Math.Max(1, movable.Count), + slot.Offset, destination, image.Length, null, UefiFvWriter.EntryName(slot.Name, slot.Type))); + } + + var packed = UefiFvParser.Parse(output, volume.Start); + var usedEnd = UefiFvParser.LiveSlots(packed).Select(slot => slot.DataEnd).DefaultIfEmpty(packed.DataStart).Max(); + UefiFvParser.WriteUsedSize(output, packed, usedEnd); + VerifyPayloads(image, output, volume.Start); + options.CancellationToken.ThrowIfCancellationRequested(); + Commit(archive, output); + options.OnProgress?.Invoke(new DefragProgressEvent("complete", 1, -1, -1, output.Length, null, "UEFI FV defragmented")); + } + + internal static void Purge(Stream archive) { + var (image, volume) = OpenWritable(archive); + image.AsSpan(volume.DataStart, volume.End - volume.DataStart).Fill(volume.EraseByte); + UefiFvParser.WriteUsedSize(image, volume, volume.DataStart); + Commit(archive, image); + } + + internal static (byte[] Image, UefiFvLayout Volume) Read(Stream archive, bool writable) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanSeek || writable && !archive.CanWrite) + throw new ArgumentException(writable ? "UEFI FV maintenance requires a readable/writable/seekable stream." : "UEFI FV layout requires a readable/seekable stream.", nameof(archive)); + if (archive.Length > int.MaxValue) + throw new NotSupportedException("The in-memory UEFI FV implementation supports images up to 2 GiB."); + archive.Position = 0; + var image = new byte[checked((int)archive.Length)]; + archive.ReadExactly(image); + var start = UefiFvReader.FindFirst(image) ?? throw new InvalidDataException("UEFI FV header was not found."); + return (image, UefiFvParser.Parse(image, start)); + } + + internal static void EnsureMutable(UefiFvLayout volume) { + if (volume.FileSystemGuid != UefiFvConstants.Ffs2Guid && volume.FileSystemGuid != UefiFvConstants.Ffs3Guid) + throw new NotSupportedException($"UEFI FV uses unsupported file-system GUID {volume.FileSystemGuid:D}."); + if (volume.IsSigned) + throw new NotSupportedException("Signed UEFI firmware volumes cannot be modified without re-signing the complete volume."); + } + + internal static void Commit(Stream archive, byte[] image) { + archive.Position = 0; + archive.Write(image); + archive.SetLength(image.Length); + } + + private static (byte[] Image, UefiFvLayout Volume) OpenWritable(Stream archive) { + var result = Read(archive, writable: true); + EnsureMutable(result.Volume); + return result; + } + + private static bool IsAnchored(UefiFvSlot slot) + => slot.IsPad || slot.IsFixed || slot.Name == UefiFvConstants.VolumeTopGuid || slot.DataAlignment > 1; + + private static int FindRun(byte[] image, UefiFvLayout volume, int footprint) { + for (var p = volume.DataStart; p + footprint <= volume.End; p += UefiFvWriter.Alignment) + if (UefiFvParser.IsErased(image.AsSpan(p, footprint), volume.EraseByte)) return p; + return -1; + } + + private static void VerifyPayloads(byte[] before, byte[] after, int start) { + var expected = UefiFvReader.Read(before, start).Files.Where(f => f.Type != 0xF0).ToList(); + var actual = UefiFvReader.Read(after, start).Files.Where(f => f.Type != 0xF0).ToList(); + if (expected.Count != actual.Count) throw new InvalidDataException("UEFI FV defrag changed the live-file count."); + foreach (var file in expected) { + var index = actual.FindIndex(candidate => candidate.Name == file.Name && candidate.Type == file.Type + && candidate.Contents.AsSpan().SequenceEqual(file.Contents)); + if (index < 0) throw new InvalidDataException($"UEFI FV defrag changed payload {file.Name:D}."); + actual.RemoveAt(index); + } + } +} diff --git a/FileFormats/FileFormat.UefiFv/UefiFvParser.cs b/FileFormats/FileFormat.UefiFv/UefiFvParser.cs new file mode 100644 index 000000000..bbf4f99f3 --- /dev/null +++ b/FileFormats/FileFormat.UefiFv/UefiFvParser.cs @@ -0,0 +1,105 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; + +namespace FileFormat.UefiFv; + +internal static class UefiFvParser { + internal static UefiFvLayout Parse(ReadOnlySpan data, int start) { + if (start < 0 || start > data.Length - 56) + throw new InvalidDataException("UefiFv: file shorter than minimum FV header."); + if (!data.Slice(start + UefiFvReader.SignatureOffset, 4).SequenceEqual(UefiFvReader.Signature)) + throw new InvalidDataException($"UefiFv: '_FVH' signature not found at offset {start + UefiFvReader.SignatureOffset}."); + + var fsGuid = new Guid(data.Slice(start + 16, 16)); + var length64 = BinaryPrimitives.ReadUInt64LittleEndian(data[(start + 32)..]); + if (length64 < 56 || length64 > int.MaxValue) + throw new NotSupportedException($"UefiFv: unsupported firmware-volume length {length64} bytes."); + var end = checked(start + (int)length64); + if (end > data.Length) throw new InvalidDataException("UefiFv: firmware volume extends past the image."); + + var attributes = BinaryPrimitives.ReadUInt32LittleEndian(data[(start + 44)..]); + var erasePolarity = (attributes & UefiFvConstants.ErasePolarityMask) != 0; + var eraseByte = erasePolarity ? (byte)0xFF : (byte)0x00; + var headerLength = BinaryPrimitives.ReadUInt16LittleEndian(data[(start + 48)..]); + if (headerLength < 56 || start + headerLength > end) throw new InvalidDataException("UefiFv: invalid header length."); + var dataStart = start + headerLength; + int? usedSizeOffset = null; + var signed = false; + + var extOffset = BinaryPrimitives.ReadUInt16LittleEndian(data[(start + 52)..]); + if (extOffset != 0) { + var ext = checked(start + extOffset); + if (ext < start || ext > end - 20) throw new InvalidDataException("UefiFv: invalid extended-header offset."); + var extSizeRaw = BinaryPrimitives.ReadUInt32LittleEndian(data[(ext + 16)..]); + if (extSizeRaw < 20 || extSizeRaw > int.MaxValue) throw new InvalidDataException("UefiFv: invalid extended-header size."); + var extEnd = checked(ext + (int)extSizeRaw); + if (extEnd > end) throw new InvalidDataException("UefiFv: extended header extends past the volume."); + dataStart = Math.Max(dataStart, extEnd); + for (var p = ext + 20; p + 4 <= extEnd;) { + var entrySize = BinaryPrimitives.ReadUInt16LittleEndian(data[p..]); + var entryType = BinaryPrimitives.ReadUInt16LittleEndian(data[(p + 2)..]); + if (entrySize < 4 || p + entrySize > extEnd) throw new InvalidDataException("UefiFv: invalid extension entry."); + if (entryType == 3 && entrySize >= 8) usedSizeOffset = p + 4; + if (entryType == 2 && entrySize >= 20 && new Guid(data.Slice(p + 4, 16)) == UefiFvConstants.SignedContentsGuid) + signed = true; + p += entrySize; + } + } + + dataStart = UefiFvConstants.AlignOffset(start, dataStart); + if (dataStart > end) throw new InvalidDataException("UefiFv: file area lies beyond the volume."); + var slots = ReadSlots(data, start, end, dataStart, erasePolarity, eraseByte); + return new UefiFvLayout(start, end, dataStart, fsGuid, attributes, erasePolarity, eraseByte, usedSizeOffset, signed, slots); + } + + internal static IEnumerable LiveSlots(UefiFvLayout volume, bool includePad = true) { + foreach (var slot in volume.Slots) + if ((includePad || !slot.IsPad) && IsLive(volume, slot)) yield return slot; + } + + internal static bool IsLive(UefiFvLayout volume, UefiFvSlot slot) { + if (slot.State == UefiFvConstants.DataValid) return true; + if (slot.State != UefiFvConstants.MarkedForUpdate) return false; + return !volume.Slots.Any(other => other.Offset > slot.Offset && other.Name == slot.Name && other.State == UefiFvConstants.DataValid); + } + + internal static bool IsDiscardable(UefiFvLayout volume, UefiFvSlot slot) + => slot.State is UefiFvConstants.Deleted or UefiFvConstants.HeaderInvalid + || slot.State == UefiFvConstants.MarkedForUpdate && !IsLive(volume, slot); + + internal static bool IsErased(ReadOnlySpan data, byte eraseByte) { + foreach (var value in data) if (value != eraseByte) return false; + return true; + } + + internal static void WriteUsedSize(Span image, UefiFvLayout volume, int usedEnd) { + if (volume.UsedSizeOffset is { } offset) + BinaryPrimitives.WriteUInt32LittleEndian(image[offset..], checked((uint)(usedEnd - volume.Start))); + } + + private static List ReadSlots(ReadOnlySpan data, int start, int end, int dataStart, + bool erasePolarity, byte eraseByte) { + var result = new List(); + for (var p = dataStart; p + 24 <= end;) { + var quantum = Math.Min(8, end - p); + if (IsErased(data.Slice(p, quantum), eraseByte)) { p += quantum; continue; } + var attr = data[p + 19]; + var large = (attr & UefiFvConstants.LargeFile) != 0; + var headerSize = large ? 32 : 24; + if (p + headerSize > end) throw new InvalidDataException("UefiFv: truncated FFS header."); + var size24 = (uint)(data[p + 20] | (data[p + 21] << 8) | (data[p + 22] << 16)); + var size64 = large ? BinaryPrimitives.ReadUInt64LittleEndian(data[(p + 24)..]) : size24; + if (large && size24 != 0) throw new InvalidDataException("UefiFv: large FFS header has non-zero Size[3]."); + if (size64 < (ulong)headerSize || size64 > int.MaxValue || p + (long)size64 > end) + throw new InvalidDataException($"UefiFv: invalid FFS size at FV offset 0x{p - start:X}."); + var size = (int)size64; + var aligned = UefiFvConstants.Align8(size); + var footprint = p + (long)aligned <= end ? aligned : size; + var rawState = data[p + 23]; + result.Add(new UefiFvSlot(p, headerSize, size, footprint, new Guid(data.Slice(p, 16)), data[p + 18], attr, + rawState, UefiFvConstants.EffectiveState(rawState, erasePolarity))); + p = checked(p + footprint); + } + return result; + } +} diff --git a/FileFormats/FileFormat.UefiFv/UefiFvReader.cs b/FileFormats/FileFormat.UefiFv/UefiFvReader.cs index 5e57d91bc..ec09bd5fe 100644 --- a/FileFormats/FileFormat.UefiFv/UefiFvReader.cs +++ b/FileFormats/FileFormat.UefiFv/UefiFvReader.cs @@ -1,15 +1,9 @@ #pragma warning disable CS1591 using System.Buffers.Binary; -using System.Text; namespace FileFormat.UefiFv; -/// -/// Reader for UEFI Platform Initialization (PI) Firmware Volumes. Locates the -/// FV header by scanning for the _FVH signature at offset 40 from the -/// start of each 16-byte-aligned candidate (UEFI PI Volume 3). Walks the FFS -/// file list and returns one record per live file. -/// +/// Reader for UEFI PI firmware volumes and standard FFS2/FFS3 file records. public sealed class UefiFvReader { public static readonly byte[] Signature = [(byte)'_', (byte)'F', (byte)'V', (byte)'H']; public const int SignatureOffset = 40; @@ -23,33 +17,16 @@ public sealed record FvHeader( ushort ExtHeaderOffset, byte Revision, IReadOnlyList<(uint NumBlocks, uint Length)> BlockMap - ); - - public sealed record FfsFile( - Guid Name, - byte Type, - byte Attributes, - byte State, - uint Size, - byte[] Contents - ); + ) { + public bool ErasePolarity => (this.Attributes & UefiFvConstants.ErasePolarityMask) != 0; + public byte EraseByte => this.ErasePolarity ? (byte)0xFF : (byte)0x00; + } - public sealed record FirmwareVolume( - int StartOffset, - FvHeader Header, - IReadOnlyList Files - ); + public sealed record FfsFile(Guid Name, byte Type, byte Attributes, byte State, uint Size, byte[] Contents); + public sealed record FirmwareVolume(int StartOffset, FvHeader Header, IReadOnlyList Files); public static FirmwareVolume Read(ReadOnlySpan data, int fvStart = 0) { - if (data.Length < fvStart + 56) - throw new InvalidDataException("UefiFv: file shorter than minimum FV header."); - - var sigSpan = data.Slice(fvStart + SignatureOffset, 4); - if (!sigSpan.SequenceEqual(Signature)) - throw new InvalidDataException( - $"UefiFv: '_FVH' signature not found at offset {fvStart + SignatureOffset}."); - - var fsGuid = new Guid(data.Slice(fvStart + 16, 16)); + var layout = UefiFvParser.Parse(data, fvStart); var fvLength = BinaryPrimitives.ReadUInt64LittleEndian(data[(fvStart + 32)..]); var attributes = BinaryPrimitives.ReadUInt32LittleEndian(data[(fvStart + 44)..]); var headerLength = BinaryPrimitives.ReadUInt16LittleEndian(data[(fvStart + 48)..]); @@ -58,71 +35,30 @@ public static FirmwareVolume Read(ReadOnlySpan data, int fvStart = 0) { var revision = data[fvStart + 55]; var blockMap = new List<(uint, uint)>(); - var p = fvStart + 56; - while (p + 8 <= data.Length) { - var nb = BinaryPrimitives.ReadUInt32LittleEndian(data[p..]); - var bl = BinaryPrimitives.ReadUInt32LittleEndian(data[(p + 4)..]); - p += 8; - if (nb == 0 && bl == 0) break; - blockMap.Add((nb, bl)); + var headerEnd = checked(fvStart + headerLength); + var terminated = false; + for (var p = fvStart + 56; p + 8 <= headerEnd; p += 8) { + var blocks = BinaryPrimitives.ReadUInt32LittleEndian(data[p..]); + var length = BinaryPrimitives.ReadUInt32LittleEndian(data[(p + 4)..]); + if (blocks == 0 && length == 0) { terminated = true; break; } + blockMap.Add((blocks, length)); } + if (!terminated) throw new InvalidDataException("UefiFv: firmware-volume block map is not terminated inside HeaderLength."); - var header = new FvHeader(fsGuid, fvLength, attributes, headerLength, checksum, extOff, revision, blockMap); - var ffsStart = fvStart + headerLength; - var ffsEnd = checked((int)Math.Min((long)data.Length, fvStart + (long)fvLength)); - var files = ReadFfsFiles(data, ffsStart, ffsEnd); + var files = UefiFvParser.LiveSlots(layout).Select(slot => new FfsFile( + slot.Name, slot.Type, slot.Attributes, slot.RawState, checked((uint)slot.Size), + data.Slice(slot.DataOffset, slot.DataLength).ToArray())).ToList(); + var header = new FvHeader(new Guid(data.Slice(fvStart + 16, 16)), fvLength, attributes, + headerLength, checksum, extOff, revision, blockMap); return new FirmwareVolume(fvStart, header, files); } public static int? FindFirst(ReadOnlySpan data) { - for (var i = 0; i + SignatureOffset + 4 <= data.Length; i += 16) { - if (data.Slice(i + SignatureOffset, 4).SequenceEqual(Signature)) - return i; - } + for (var i = 0; i + SignatureOffset + 4 <= data.Length; i += 16) + if (data.Slice(i + SignatureOffset, 4).SequenceEqual(Signature)) return i; return null; } - private static List ReadFfsFiles(ReadOnlySpan data, int start, int end) { - var files = new List(); - var pos = Align8(start); - while (pos + 24 <= end) { - // Free/deleted regions may occur between live files after offline - // mutation, so a gap has to be walked over one alignment quantum at a - // time. Only the quantum itself is tested: asking whether a whole 24-byte - // header is erased reads past the end of the gap, and the last two - // quanta of a gap then look half-written — the walk parsed that overlap - // as a file, took its size from the middle of the real header's GUID, and - // gave up on the volume. An FFS file starts on this boundary and opens - // with its GUID, which is never all-ones. - if (IsErased(data.Slice(pos, 8))) { - pos += 8; - continue; - } - - var header = data.Slice(pos, 24); - - var name = new Guid(header[..16]); - var type = header[18]; - var attrs = header[19]; - var size = (uint)(header[20] | (header[21] << 8) | (header[22] << 16)); - var state = header[23]; - if (size < 24 || pos + (long)size > end) break; - - var contents = data.Slice(pos + 24, checked((int)size - 24)).ToArray(); - files.Add(new FfsFile(name, type, attrs, state, size, contents)); - pos = Align8(pos + checked((int)size)); - } - return files; - - static bool IsErased(ReadOnlySpan bytes) { - foreach (var b in bytes) - if (b != 0xFF) return false; - return true; - } - - static int Align8(int v) => (v + 7) & ~7; - } - public static string FileTypeName(byte t) => t switch { 0x00 => "EFI_FV_FILETYPE_ALL", 0x01 => "EFI_FV_FILETYPE_RAW", @@ -145,8 +81,8 @@ static bool IsErased(ReadOnlySpan bytes) { }; public static string ShortTypeTag(byte t) { - var n = FileTypeName(t); + var name = FileTypeName(t); const string prefix = "EFI_FV_FILETYPE_"; - return n.StartsWith(prefix, StringComparison.Ordinal) ? n[prefix.Length..] : n; + return name.StartsWith(prefix, StringComparison.Ordinal) ? name[prefix.Length..] : name; } } diff --git a/FileFormats/FileFormat.UefiFv/UefiFvWriter.cs b/FileFormats/FileFormat.UefiFv/UefiFvWriter.cs index cc6a148da..90c2cd305 100644 --- a/FileFormats/FileFormat.UefiFv/UefiFvWriter.cs +++ b/FileFormats/FileFormat.UefiFv/UefiFvWriter.cs @@ -5,28 +5,32 @@ namespace FileFormat.UefiFv; -/// Writes a standalone PI firmware volume containing ordinary FFS2 files. +/// Writes standalone PI firmware volumes containing ordinary FFS2/FFS3 files. internal static class UefiFvWriter { internal const int HeaderLength = 72; internal const int FfsHeaderLength = 24; + internal const int FfsHeader2Length = 32; internal const int Alignment = 8; internal const int DefaultReserve = 64 * 1024; internal const uint FvAttributes = 0x0004FEFF; - internal static readonly Guid Ffs2Guid = Guid.Parse("8C8CE578-8A3D-4F1C-9935-896185C32DD3"); + internal static readonly Guid Ffs2Guid = UefiFvConstants.Ffs2Guid; + internal static readonly Guid Ffs3Guid = UefiFvConstants.Ffs3Guid; internal readonly record struct FileIdentity(Guid Guid, byte Type); public static byte[] Build(IEnumerable<(string Name, byte[] Data)> inputs) { var files = inputs.Select(i => (Identity: IdentityFromName(i.Name), i.Data)).ToList(); - var used = HeaderLength + files.Sum(f => Align8(checked(FfsHeaderLength + f.Data.Length))); + var useFfs3 = files.Any(f => RequiresLargeFile(f.Data.Length)); + var fileSystemGuid = useFfs3 ? Ffs3Guid : Ffs2Guid; + var used = HeaderLength + files.Sum(f => Align8(checked(HeaderSize(f.Data.Length) + f.Data.Length))); var capacity = Align4K(checked(used + DefaultReserve)); var image = new byte[capacity]; image.AsSpan().Fill(0xFF); - WriteVolumeHeader(image); + WriteVolumeHeader(image, fileSystemGuid); var position = HeaderLength; foreach (var file in files) { - var encoded = BuildFfsFile(file.Identity.Guid, file.Identity.Type, file.Data); + var encoded = BuildFfsFile(file.Identity.Guid, file.Identity.Type, file.Data, fileSystemGuid, erasePolarity: true); encoded.CopyTo(image, position); position += Align8(encoded.Length); } @@ -42,32 +46,44 @@ internal static FileIdentity IdentityFromName(string name) { } var hash = SHA256.HashData(Encoding.UTF8.GetBytes(name.Replace('\\', '/'))); - return new FileIdentity(new Guid(hash.AsSpan(0, 16)), 0x01); // RAW + return new FileIdentity(new Guid(hash.AsSpan(0, 16)), 0x01); } - internal static byte[] BuildFfsFile(Guid guid, byte type, ReadOnlySpan contents) { - var size = checked(FfsHeaderLength + contents.Length); - if (size > 0xFFFFFF) - throw new NotSupportedException("FFS2 files larger than 16 MiB require EFI_FFS_FILE_HEADER2."); + internal static byte[] BuildFfsFile(Guid guid, byte type, ReadOnlySpan contents) + => BuildFfsFile(guid, type, contents, Ffs2Guid, erasePolarity: true); + internal static byte[] BuildFfsFile(Guid guid, byte type, ReadOnlySpan contents, + Guid fileSystemGuid, bool erasePolarity) { + var large = RequiresLargeFile(contents.Length); + if (large && fileSystemGuid != Ffs3Guid) + throw new NotSupportedException("FFS files larger than 16 MiB require an EFI_FIRMWARE_FILE_SYSTEM3_GUID volume."); + + var headerSize = large ? FfsHeader2Length : FfsHeaderLength; + var size = checked(headerSize + contents.Length); var file = new byte[size]; guid.TryWriteBytes(file.AsSpan(0, 16)); - file[17] = 0xAA; // fixed checksum when FFS_ATTRIB_CHECKSUM is clear + file[17] = 0xAA; file[18] = type; - file[19] = 0; - file[20] = (byte)size; - file[21] = (byte)(size >> 8); - file[22] = (byte)(size >> 16); - file[23] = 0xF8; - contents.CopyTo(file.AsSpan(FfsHeaderLength)); - file[16] = HeaderChecksum(file.AsSpan(0, FfsHeaderLength)); + file[19] = large ? UefiFvConstants.LargeFile : (byte)0; + if (large) + BinaryPrimitives.WriteUInt64LittleEndian(file.AsSpan(24), (ulong)size); + else { + file[20] = (byte)size; + file[21] = (byte)(size >> 8); + file[22] = (byte)(size >> 16); + } + file[23] = UefiFvConstants.EncodeState( + UefiFvConstants.HeaderConstruction | UefiFvConstants.HeaderValid | UefiFvConstants.DataValid, + erasePolarity); + contents.CopyTo(file.AsSpan(headerSize)); + file[16] = HeaderChecksum(file.AsSpan(0, headerSize)); return file; } - internal static void WriteVolumeHeader(Span image) { + internal static void WriteVolumeHeader(Span image, Guid? fileSystemGuid = null) { if (image.Length < HeaderLength) throw new ArgumentException("FV buffer is too small.", nameof(image)); image[..HeaderLength].Clear(); - Ffs2Guid.TryWriteBytes(image.Slice(16, 16)); + (fileSystemGuid ?? Ffs2Guid).TryWriteBytes(image.Slice(16, 16)); BinaryPrimitives.WriteUInt64LittleEndian(image[32..], (ulong)image.Length); "_FVH"u8.CopyTo(image[40..]); BinaryPrimitives.WriteUInt32LittleEndian(image[44..], FvAttributes); @@ -98,8 +114,10 @@ private static ushort VolumeHeaderChecksum(ReadOnlySpan header) { return unchecked((ushort)(0 - sum)); } - internal static int Align8(int value) => checked((value + Alignment - 1) & ~(Alignment - 1)); + internal static int Align8(int value) => UefiFvConstants.Align8(value); private static int Align4K(int value) => checked((value + 4095) & ~4095); + private static bool RequiresLargeFile(int dataLength) => (long)FfsHeaderLength + dataLength > 0xFFFFFF; + private static int HeaderSize(int dataLength) => RequiresLargeFile(dataLength) ? FfsHeader2Length : FfsHeaderLength; internal static string EntryName(Guid guid, byte type) => $"{guid:D}_{UefiFvReader.ShortTypeTag(type)}.bin"; @@ -107,22 +125,11 @@ internal static string EntryName(Guid guid, byte type) private static byte ParseTypeTag(string tail) { var tag = Path.GetFileNameWithoutExtension(tail).TrimStart('_').ToUpperInvariant(); return tag switch { - "RAW" => 0x01, - "FREEFORM" => 0x02, - "SECURITY_CORE" => 0x03, - "PEI_CORE" => 0x04, - "DXE_CORE" => 0x05, - "PEIM" => 0x06, - "DRIVER" => 0x07, - "COMBINED_PEIM_DRIVER" => 0x08, - "APPLICATION" => 0x09, - "MM" => 0x0A, - "FIRMWARE_VOLUME_IMAGE" => 0x0B, - "COMBINED_MM_DXE" => 0x0C, - "MM_CORE" => 0x0D, - "MM_STANDALONE" => 0x0E, - "MM_CORE_STANDALONE" => 0x0F, - _ => 0x01, + "RAW" => 0x01, "FREEFORM" => 0x02, "SECURITY_CORE" => 0x03, "PEI_CORE" => 0x04, + "DXE_CORE" => 0x05, "PEIM" => 0x06, "DRIVER" => 0x07, "COMBINED_PEIM_DRIVER" => 0x08, + "APPLICATION" => 0x09, "MM" => 0x0A, "FIRMWARE_VOLUME_IMAGE" => 0x0B, + "COMBINED_MM_DXE" => 0x0C, "MM_CORE" => 0x0D, "MM_STANDALONE" => 0x0E, + "MM_CORE_STANDALONE" => 0x0F, _ => 0x01, }; } } From 34998262a815027d76c13758758561d6dd0186ce Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:09:54 +0200 Subject: [PATCH 088/165] Parse real eCryptfs lower-file headers --- .../FileSystem.Ecryptfs/EcryptfsReader.cs | 173 +++++++----------- 1 file changed, 64 insertions(+), 109 deletions(-) diff --git a/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs b/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs index b9fed5f75..8d4db9272 100644 --- a/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs +++ b/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs @@ -1,127 +1,82 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; -using System.Globalization; -using System.Text; +using System.Security.Cryptography; namespace FileSystem.Ecryptfs; /// -/// Reads eCryptfs per-file encryption headers. eCryptfs is a stacking -/// file-level encryption filesystem (Linux) — every encrypted file is -/// stored on the lower filesystem as a regular file whose first page is -/// a metadata header followed by AES-CBC ciphertext extents. The header -/// starts with a 4-byte big-endian marker (0x3C81B7F5) so the -/// individual on-disk container is well-defined and detectable. -/// Decryption requires the user's mount passphrase + EFEK (Encrypted -/// File Encryption Key) tag-3 / tag-11 packets and is OUT OF SCOPE; this -/// reader surfaces the parsed header + the encrypted payload as a single -/// opaque entry. -/// -/// File header layout (big-endian, file offset 0): -/// 0x00 u32 marker == 0x3C81B7F5 -/// 0x04 u64 decrypted-size (plaintext length, host-endian on Linux) -/// 0x0C u32 flags -/// 0x10 u32 extent-size (typically 4096) -/// 0x14 ... EFEK packets, tag-3 / tag-11 OpenPGP-style -/// ~0x800 start of AES-CBC ciphertext extents +/// Reader for an eCryptfs lower file. eCryptfs is a stacked filesystem, so one +/// lower file represents one encrypted upper file rather than a complete volume. /// public sealed class EcryptfsReader : IDisposable { - private readonly byte[] _data; + private readonly Stream _stream; + private readonly EcryptfsCodec.Header _header; private readonly List _entries = []; - /// - /// Gets the entries. - /// - public IReadOnlyList Entries => _entries; - - /// - /// Gets or sets the marker. - /// - public uint Marker { get; private set; } - /// - /// Gets or sets the decrypted size. - /// - public ulong DecryptedSize { get; private set; } - /// - /// Gets or sets the flags. - /// - public uint Flags { get; private set; } - /// - /// Gets or sets the extent size. - /// - public uint ExtentSize { get; private set; } - /// - /// Gets a value indicating whether valid header. - /// - public bool ValidHeader { get; private set; } - - /// - /// Defines the ecryptfs marker constant value. - /// - public const uint EcryptfsMarker = 0x3C81B7F5u; - private const int HeaderMinSize = 24; - - /// - /// Initializes a new instance of . - /// public EcryptfsReader(Stream stream) { - using var ms = new MemoryStream(); - stream.CopyTo(ms); - _data = ms.ToArray(); - Parse(); - } + ArgumentNullException.ThrowIfNull(stream); + this._stream = stream; + this._header = EcryptfsCodec.ReadHeader(stream); - private void Parse() { - if (_data.Length < HeaderMinSize) - throw new InvalidDataException("Ecryptfs: file too small for header."); - - var marker = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(0, 4)); - if (marker != EcryptfsMarker) - throw new InvalidDataException($"Ecryptfs: invalid marker 0x{marker:X8} at offset 0 (expected 0x{EcryptfsMarker:X8})."); - - this.Marker = marker; - // Decrypted-size is 8 bytes BE per the eCryptfs on-disk format documentation - // (kernel writes it big-endian regardless of host byte order so files round-trip across hosts). - this.DecryptedSize = BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(4, 8)); - this.Flags = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(12, 4)); - this.ExtentSize = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(16, 4)); - this.ValidHeader = true; - - var meta = BuildMetadata(); - _entries.Add(new EcryptfsEntry { Name = "FULL.ecryptfs", Size = _data.Length, IsDirectory = false, Data = _data }); - _entries.Add(new EcryptfsEntry { Name = "metadata.ini", Size = meta.Length, IsDirectory = false, Data = meta }); - - // Surface the encrypted ciphertext as opaque blob (everything after the metadata page). - var payloadStart = (int)Math.Min((long)Math.Max(this.ExtentSize, 4096), _data.Length); - var payloadLen = _data.Length - payloadStart; - if (payloadLen > 0) { - var blob = _data.AsSpan(payloadStart, payloadLen).ToArray(); - _entries.Add(new EcryptfsEntry { Name = "ciphertext.bin", Size = blob.Length, IsDirectory = false, Data = blob }); + if (this._header.DecryptedSize > 0) { + if (this._header.DecryptedSize > long.MaxValue) + throw new InvalidDataException("eCryptfs plaintext size exceeds the supported entry-size range."); + this._entries.Add(new EcryptfsEntry { + Name = "content.bin", + Size = (long)this._header.DecryptedSize, + IsDirectory = false, + }); } } - private byte[] BuildMetadata() { - var bldr = new StringBuilder(); - bldr.Append("parse_status=ok\n"); - bldr.Append("format=eCryptfs (file-level encryption)\n"); - bldr.Append(CultureInfo.InvariantCulture, $"marker=0x{this.Marker:X8}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"decrypted_size={this.DecryptedSize}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"flags=0x{this.Flags:X8}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"extent_size={this.ExtentSize}\n"); - bldr.Append("note=Ciphertext exposed opaque; decryption requires mount passphrase + EFEK.\n"); - return Encoding.UTF8.GetBytes(bldr.ToString()); - } + /// The validated eCryptfs marker relation constant. + public uint Marker => EcryptfsCodec.MarkerMagic; + + /// The random first word of the two-word marker pair stored on disk. + public uint MarkerWord => this._header.MarkerWord; + + /// Plaintext file length stored in the lower-file header. + public ulong DecryptedSize => this._header.DecryptedSize; + + /// Raw eCryptfs file flags, including the version in the top byte. + public uint Flags => this._header.Flags; + + /// eCryptfs lower-file format version. + public byte FileVersion => this._header.FileVersion; + + /// Encryption extent size in bytes. + public uint ExtentSize => this._header.ExtentSize; - /// - /// Decodes the supplied input. - /// - public byte[] Extract(EcryptfsEntry entry) { - ArgumentNullException.ThrowIfNull(entry); - return entry.Data; + /// Number of header extents preceding ciphertext data. + public ushort HeaderExtentCount => this._header.HeaderExtentCount; + + /// Total metadata region at the beginning of the lower file. + public int MetadataSize => this._header.MetadataSize; + + /// Canonical lower-file length implied by the plaintext size and extent geometry. + public long CanonicalLength => this._header.CanonicalLength; + + /// Human-readable cipher of the first passphrase packet, when present. + public string CipherDescription => this._header.CipherDescription; + + /// Raw eight-byte authentication-token signature as lowercase hexadecimal. + public string? PassphraseSignature + => this._header.PassphrasePackets.Count == 0 + ? null + : Convert.ToHexStringLower(this._header.PassphrasePackets[0].Signature); + + public IReadOnlyList Entries => this._entries; + + /// Decrypts the single logical upper-file payload with a passphrase. + /// The passphrase does not match this lower file. + public byte[] ExtractContent(string password) { + if (this._header.DecryptedSize > int.MaxValue) + throw new IOException("This eCryptfs payload is too large for the in-memory extraction API."); + using var output = new MemoryStream((int)this._header.DecryptedSize); + EcryptfsCodec.DecryptTo(this._stream, output, this._header, password); + return output.ToArray(); } - /// - /// Releases resources held by this instance. - /// - public void Dispose() { } + public void Dispose() { + // The reader never owns the caller's stream. + } } From 420adc0145e1b0395b362b4ae1de9ed71e597b80 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:10:10 +0200 Subject: [PATCH 089/165] Promote NWFS386 to conservative R/W rebuild support --- .../Nwfs386FormatDescriptor.cs | 678 +++++++++++++++--- 1 file changed, 569 insertions(+), 109 deletions(-) diff --git a/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs b/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs index b24d9862d..b11bc0c51 100644 --- a/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs +++ b/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs @@ -1,142 +1,602 @@ #pragma warning disable CS1591 -using System.Globalization; -using System.Text; +using System.Buffers.Binary; using Compression.Registry; +using FileSystem.Nwfs; using static Compression.Registry.FormatHelpers; namespace FileSystem.Nwfs386; /// -/// Read-only descriptor for Novell NetWare 386 (NWFS386) raw partition dumps, -/// detected via the "NetW" ASCII prefix at offset 0. DOS partition type -/// 0x65. The on-disk format is FAT-like but proprietary; no parser is -/// attempted — the image is surfaced as a single opaque entry with -/// metadata.ini noting the partition-type hint. -/// -/// References: -/// -/// https://www.win.tue.nl/~aeb/partitions/partition_types-1.html — partition-type catalogue (0x65 = Novell NetWare) -/// https://en.wikipedia.org/wiki/NetWare_File_System — Wikipedia article -/// Novell NetWare 386 internal documentation — the on-disk format was never published -/// +/// Compatibility descriptor for the Novell NetWare 386 Traditional File System. +/// The actual reader/writer lives in FileSystem.Nwfs; this descriptor keeps +/// the historical Nwfs386 id/extensions while sharing that implementation. /// /// -/// Distinct from FileSystem.Nwfs, which detects via the -/// "HOTFIX00" magic at byte offset 0x4000 (sector-32-aligned NetWare -/// HOTFIX header). NWFS386 here covers raw NWFS partition dumps that -/// expose the "NetW" four-byte tag at offset 0 instead of the HOTFIX area. +/// +/// Fresh images and rewrites intentionally target the conservative profile the +/// shared writer can prove: one Traditional NetWare volume, one segment, DOS +/// namespace only, without compression, suballocation, migration, auditing or +/// special directory records. Unsupported legacy profiles remain readable where +/// the shared reader understands them, but mutation fails before the source is +/// touched. +/// +/// +/// Novell documents MBR partition type 0x65 as a Traditional NetWare +/// partition. The detailed on-disk structures are reconstructed from public +/// reverse-engineering/reference implementations; no third-party implementation +/// code is copied here. +/// /// -public sealed class Nwfs386FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { - /// - /// Gets the id. - /// +public sealed class Nwfs386FormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IArchiveInMemoryExtract, + IArchiveCreatable, + IArchiveModifiable, + IArchiveDefragmentable, + IArchiveShrinkable, + IFormatOptionsSchema, + ILayoutOptimizable { + + private const long MaxManagedImageBytes = 512L * 1024 * 1024; + private const int SectorSize = 512; + private const int DirectoryEntryBytes = 128; + private const int FatEntryBytes = 8; + private const int VolumeAreaBytes = 4 * 16384; + private const uint NoBlock = 0xFFFFFFFF; + private const uint DirIdAvailable = 0xFFFFFFFF; + private const uint DirIdGrantOrDeleted = 0xFFFFFFFE; + private const uint DirIdVolumeInfo = 0xFFFFFFFD; + private const uint HighestSpecialDirectoryId = 0xFFFFFF00; + private const uint AttributeDirectory = 0x10; + private const uint AttributeArchive = 0x20; + + private static readonly int[] SupportedBlockSizes = + [1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144]; + public string Id => "Nwfs386"; - /// - /// Gets the display name. - /// - public string DisplayName => "NWFS386 (Novell NetWare 386 raw)"; - /// - /// Gets the category. - /// + public string DisplayName => "NWFS386 (Novell Traditional NetWare filesystem)"; public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest; - /// - /// Gets the default extension. - /// + FormatCapabilities.CanList | + FormatCapabilities.CanExtract | + FormatCapabilities.CanCreate | + FormatCapabilities.CanModify | + FormatCapabilities.CanTest | + FormatCapabilities.SupportsMultipleEntries | + FormatCapabilities.SupportsDirectories; + public string DefaultExtension => ".nwfs386"; - /// - /// Gets the extensions. - /// public IReadOnlyList Extensions => [".nwfs386", ".nw386"]; - /// - /// Gets the compound extensions. - /// public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// - public IReadOnlyList MagicSignatures => [ - // "NetW" ASCII at offset 0 — proprietary marker present in some NWFS386 - // raw partition dumps. Confidence 0.60 — short ASCII string at fixed - // offset, not from a published vendor spec. - new([0x4E, 0x65, 0x74, 0x57], Offset: 0, Confidence: 0.60), - ]; - /// - /// Gets the methods. - /// + + // The real HOTFIX00 signatures are already owned by FileSystem.Nwfs. + // Registering them twice would make magic routing depend on registry order. + // Nwfs386 is therefore an extension-routed compatibility id, exactly like + // other aliases in the repository that share one physical format. + public IReadOnlyList MagicSignatures => []; + public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// - public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// - public string Description => "Novell NetWare 386 raw partition — opaque single-entry surface."; - - /// - /// Lists the entries in the supplied container. - /// + public string Description => + "Novell NetWare 386 Traditional filesystem. R/W rebuilds are limited to the " + + "single-segment DOS-namespace profile without compression/suballocation; " + + "advanced legacy profiles are rejected before mutation."; + + public IReadOnlyList OptionsSchema => + [ + new( + "BlockSize", + "Block size", + FormatOptionKind.Integer, + "4096", + SupportedBlockSizes.Select(static value => value.ToString(System.Globalization.CultureInfo.InvariantCulture)).ToArray(), + "Allocation block size in bytes. Changing it rebuilds the volume; existing data is preserved byte-for-byte."), + new( + "VolumeName", + "Volume name", + FormatOptionKind.String, + "SYS", + Description: "NetWare volume name (1-19 ASCII characters).") + ]; + public List List(Stream stream, string? password) { - var entries = new List(); - byte[] image; - try { - image = ReadAll(stream); - } catch { - entries.Add(new ArchiveEntryInfo(0, "FULL.nwfs386", 0, 0, "stored", false, false, null)); - entries.Add(new ArchiveEntryInfo(1, "metadata.ini", 0, 0, "stored", false, false, null)); - return entries; + var opened = Open(stream); + var result = new List(); + var index = 0; + foreach (var item in opened.Volume.List()) + result.Add(new ArchiveEntryInfo( + index++, + item.Path, + item.Length, + item.Length, + "stored", + item.IsDirectory, + false, + null)); + return result; + } + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + var opened = Open(stream); + Directory.CreateDirectory(outputDir); + + foreach (var item in opened.Volume.List()) { + if (item.IsDirectory) continue; + if (files is { Length: > 0 } && !MatchesFilter(item.Path, files)) continue; + WriteFile(outputDir, item.Path, opened.Volume.Read(item)); } + } + + public void ExtractEntry(Stream input, string entryName, Stream output, string? password) { + ArgumentNullException.ThrowIfNull(output); + var opened = Open(input); + var item = opened.Volume.List().FirstOrDefault(candidate => + !candidate.IsDirectory && + candidate.Path.Equals(entryName, StringComparison.OrdinalIgnoreCase)); - entries.Add(new ArchiveEntryInfo(0, "FULL.nwfs386", image.LongLength, image.LongLength, "stored", false, false, null)); - entries.Add(new ArchiveEntryInfo(1, "metadata.ini", 0, 0, "stored", false, false, null)); - return entries; + if (item is null) + throw new FileNotFoundException($"NWFS386 entry '{entryName}' was not found.", entryName); + + output.Write(opened.Volume.Read(item)); } - /// - /// Decodes the supplied input. - /// - public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - byte[] image; - try { - image = ReadAll(stream); - } catch { - WriteFile(outputDir, "metadata.ini", Encoding.UTF8.GetBytes("parse_status=partial\n")); - return; + public Stream OpenEntry(Stream archive, string entryName, string? password) { + var opened = Open(archive); + var item = opened.Volume.List().FirstOrDefault(candidate => + !candidate.IsDirectory && + candidate.Path.Equals(entryName, StringComparison.OrdinalIgnoreCase)); + + if (item is null) + throw new FileNotFoundException($"NWFS386 entry '{entryName}' was not found.", entryName); + + return new MemoryStream(opened.Volume.Read(item), writable: false); + } + + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + ArgumentNullException.ThrowIfNull(options); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("NWFS386 creation requires a writable, seekable stream.", nameof(output)); + + var blockSize = options.GetOptionInt( + "BlockSize", + options.GetOptionInt("ClusterSize", 4096)); + var volumeName = options.GetOption("VolumeName", "SYS"); + + var files = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in inputs) { + if (input.IsDirectory) continue; + var name = NormalizePath(input.ArchiveName); + if (name.Length == 0) + throw new InvalidDataException("NWFS386 file entries need a non-empty path."); + files[name] = input.ReadContent(); } - var ok = image.Length >= 4 && image[0] == 'N' && image[1] == 'e' && image[2] == 't' && image[3] == 'W'; - WriteIfMatch(outputDir, "FULL.nwfs386", image, files); + var image = BuildImage(files, blockSize, volumeName); + output.Position = 0; + output.SetLength(0); + output.Write(image); + output.Position = 0; + } + + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(inputs); + var opened = OpenWritable(archive); + var files = SnapshotFiles(opened.Volume); + + foreach (var input in inputs) { + if (input.IsDirectory) continue; + var name = NormalizePath(input.ArchiveName); + if (name.Length == 0) + throw new InvalidDataException("NWFS386 file entries need a non-empty path."); + files[name] = input.ReadContent(); + } - var bldr = new StringBuilder(); - bldr.Append(CultureInfo.InvariantCulture, $"parse_status={(ok ? "ok" : "partial")}\n"); - bldr.Append("magic_text=NetW\n"); - bldr.Append("dos_partition_type=0x65\n"); - bldr.Append("note=NWFS386 raw partition. Layout proprietary; surfaced as opaque blob.\n"); - WriteIfMatch(outputDir, "metadata.ini", Encoding.UTF8.GetBytes(bldr.ToString()), files); + CommitRebuild(archive, files, opened.Volume.BlockSize, opened.Volume.VolumeName); } - private static void WriteIfMatch(string outputDir, string name, byte[] data, string[]? filter) { - if (filter != null && filter.Length > 0 && !MatchesFilter(name, filter)) return; - WriteFile(outputDir, name, data); + public void Remove(Stream archive, string[] entryNames) { + var opened = OpenWritable(archive); + var files = SnapshotFiles(opened.Volume); + var removals = (entryNames ?? []) + .Select(NormalizePath) + .Where(static name => name.Length > 0) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + foreach (var name in files.Keys.ToArray()) + if (removals.Any(removal => + name.Equals(removal, StringComparison.OrdinalIgnoreCase) || + name.StartsWith(removal + "/", StringComparison.OrdinalIgnoreCase))) + files.Remove(name); + + CommitRebuild(archive, files, opened.Volume.BlockSize, opened.Volume.VolumeName); + } + + public void Defragment(Stream archive) { + var opened = OpenWritable(archive); + CommitRebuild( + archive, + SnapshotFiles(opened.Volume), + opened.Volume.BlockSize, + opened.Volume.VolumeName); + } + + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(output); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("NWFS386 shrink requires a writable, seekable target stream.", nameof(output)); + + var opened = OpenWritable(input); + var files = SnapshotFiles(opened.Volume); + var rebuilt = BuildImage(files, opened.Volume.BlockSize, opened.Volume.VolumeName); + VerifyImage(rebuilt, files, opened.Volume.BlockSize, opened.Volume.VolumeName); + + output.Position = 0; + output.SetLength(0); + if (rebuilt.LongLength >= opened.Image.LongLength) + output.Write(opened.Image); + else + output.Write(rebuilt); + output.Position = 0; } - private const int HeaderReadCap = 64 * 1024; + public LayoutAnalysis AnalyzeLayout(Stream image) { + var opened = OpenWritable(image); + var files = opened.Volume.List().Where(static item => !item.IsDirectory).ToArray(); + + static long Slack(IEnumerable items, int blockSize) + => items.Sum(item => item.Length == 0 + ? 0 + : ((item.Length + blockSize - 1) / blockSize) * blockSize - item.Length); + + var currentSlack = Slack(files, opened.Volume.BlockSize); + var optimal = SupportedBlockSizes + .Select(blockSize => (BlockSize: blockSize, Slack: Slack(files, blockSize))) + .MinBy(static candidate => (candidate.Slack, candidate.BlockSize)); + + return new LayoutAnalysis { + ImageSize = opened.Image.LongLength, + CurrentUnitSize = opened.Volume.BlockSize, + CurrentSlackBytes = currentSlack, + OptimalUnitSize = optimal.BlockSize, + OptimalSlackBytes = optimal.Slack, + RequiresRebuild = optimal.BlockSize == opened.Volume.BlockSize + ? [] + : ["BlockSize"], + Notes = [ + "Optimization compares allocation slack for the plain NWFS386 profile; changing block size is a verified rebuild.", + "Compression, suballocation, alternate namespaces and multi-segment volumes are deliberately outside the writable profile." + ], + }; + } + + public void RebuildStreaming(Stream source, Stream target, LayoutRebuildOptions options) { + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(options); + if (!target.CanWrite || !target.CanSeek) + throw new ArgumentException("NWFS386 layout rebuild requires a writable, seekable target stream.", nameof(target)); + if (options.MakeSparse || options.DeduplicateWithLinks) + throw new NotSupportedException("NWFS386 layout rebuild does not support sparse-file or hard-link transforms."); + + var opened = OpenWritable(source); + var blockSize = options.UnitSize > 0 ? options.UnitSize : opened.Volume.BlockSize; + var volumeName = opened.Volume.VolumeName; - private static byte[] ReadAll(Stream stream) { - using var ms = new MemoryStream(); - var buf = new byte[8192]; - int read; - while (ms.Length < HeaderReadCap && (read = stream.Read(buf, 0, buf.Length)) > 0) - ms.Write(buf, 0, read); - return ms.ToArray(); + if (options.Parameters is not null) { + if (options.Parameters.TryGetValue("ClusterSize", out var clusterText) && + int.TryParse(clusterText, System.Globalization.CultureInfo.InvariantCulture, out var clusterSize)) + blockSize = clusterSize; + if (options.Parameters.TryGetValue("BlockSize", out var blockText) && + int.TryParse(blockText, System.Globalization.CultureInfo.InvariantCulture, out var explicitBlockSize)) + blockSize = explicitBlockSize; + if (options.Parameters.TryGetValue("VolumeName", out var explicitVolumeName)) + volumeName = explicitVolumeName; + } + + var files = SnapshotFiles(opened.Volume); + var rebuilt = BuildImage(files, blockSize, volumeName); + VerifyImage(rebuilt, files, blockSize, volumeName); + + target.Position = 0; + target.SetLength(0); + target.Write(rebuilt); + target.Position = 0; + options.OnProgress?.Invoke(opened.Image.LongLength, opened.Image.LongLength); + } + + private static OpenedVolume Open(Stream stream) { + var image = ReadImage(stream); + var volume = NwfsReader.TryOpen(image) + ?? throw new InvalidDataException("The stream is not a readable NWFS386 Traditional NetWare volume."); + return new OpenedVolume(image, volume); + } + + private static OpenedVolume OpenWritable(Stream stream) { + var opened = Open(stream); + ValidatePlainWritableProfile(opened.Image, opened.Volume); + return opened; + } + + private static byte[] ReadImage(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead || !stream.CanSeek) + throw new ArgumentException("NWFS386 requires a readable, seekable stream.", nameof(stream)); + if (stream.Length > MaxManagedImageBytes) + throw new NotSupportedException( + $"NWFS386's shared managed reader currently caps images at {MaxManagedImageBytes:N0} bytes."); + + stream.Position = 0; + var image = new byte[checked((int)stream.Length)]; + stream.ReadExactly(image); + return image; + } + + private static Dictionary SnapshotFiles(NwfsReader volume) { + var files = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var item in volume.List().Where(static item => !item.IsDirectory)) { + if (!files.TryAdd(item.Path, volume.Read(item))) + throw new NotSupportedException($"NWFS386 writable profile rejects duplicate path '{item.Path}'."); + } + return files; } + + private static byte[] BuildImage( + IReadOnlyDictionary files, + int blockSize, + string volumeName) { + if (string.IsNullOrWhiteSpace(volumeName) || volumeName.Length > 19 || + volumeName.Any(static ch => ch is < ' ' or > '~')) + throw new InvalidDataException("NWFS386 volume name must be 1-19 printable ASCII characters."); + + var writer = new NwfsWriter { + BlockSize = blockSize, + VolumeName = volumeName, + }; + + foreach (var (path, data) in files.OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase)) + writer.AddFile(path, data); + + return writer.Build(); + } + + private static void CommitRebuild( + Stream archive, + IReadOnlyDictionary files, + int blockSize, + string volumeName) { + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("NWFS386 mutation requires a readable, writable, seekable stream.", nameof(archive)); + + var rebuilt = BuildImage(files, blockSize, volumeName); + VerifyImage(rebuilt, files, blockSize, volumeName); + + archive.Position = 0; + archive.SetLength(0); + archive.Write(rebuilt); + archive.Flush(); + archive.Position = 0; + } + + private static void VerifyImage( + byte[] image, + IReadOnlyDictionary expectedFiles, + int blockSize, + string volumeName) { + var verify = NwfsReader.TryOpen(image) + ?? throw new InvalidOperationException("NWFS386 writer produced an image its own reader rejects."); + if (verify.BlockSize != blockSize) + throw new InvalidOperationException( + $"NWFS386 rebuild changed block size ({blockSize} -> {verify.BlockSize})."); + if (!verify.VolumeName.Equals(volumeName, StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException( + $"NWFS386 rebuild changed volume name ('{volumeName}' -> '{verify.VolumeName}')."); + + var actual = SnapshotFiles(verify); + if (!actual.Keys.Order(StringComparer.OrdinalIgnoreCase) + .SequenceEqual(expectedFiles.Keys.Order(StringComparer.OrdinalIgnoreCase), StringComparer.OrdinalIgnoreCase)) + throw new InvalidOperationException("NWFS386 rebuild changed the live file set."); + + foreach (var (name, expected) in expectedFiles) + if (!actual.TryGetValue(name, out var bytes) || !bytes.AsSpan().SequenceEqual(expected)) + throw new InvalidOperationException($"NWFS386 rebuild changed the bytes of '{name}'."); + } + + private static void ValidatePlainWritableProfile(byte[] image, NwfsReader volume) { + if (!TryLocateVolume(image, out var layout)) + throw new NotSupportedException("NWFS386 mutation requires a single Traditional NetWare volume."); + + var root = layout.RootDirectoryBlock; + var guard = 0; + var sawVolumeInfo = false; + while (root != NoBlock && guard++ < 1 << 20) { + var blockOffset = layout.DataAreaOffset + (long)(root - layout.FirstSegmentBlock) * layout.BlockSize; + if (blockOffset < 0 || blockOffset + layout.BlockSize > image.LongLength) + throw new NotSupportedException("NWFS386 mutation refused a directory chain outside the supported volume bounds."); + + var entriesPerBlock = layout.BlockSize / DirectoryEntryBytes; + for (var i = 0; i < entriesPerBlock; ++i) { + var offset = checked((int)(blockOffset + (long)i * DirectoryEntryBytes)); + var entry = image.AsSpan(offset, DirectoryEntryBytes); + var parent = BinaryPrimitives.ReadUInt32LittleEndian(entry); + + if (parent == DirIdAvailable) + continue; + + if (parent == DirIdVolumeInfo) { + sawVolumeInfo = true; + // ROOT/ROOT3X: byte 23 is VolumeFlags. Writer-created plain volumes keep + // auditing, suballocation, compression, migration, NDS/trustee flags off. + if (entry[23] != 0) + throw new NotSupportedException( + "NWFS386 mutation is limited to volumes without compression, suballocation, migration, auditing or NDS flags."); + continue; + } + + // -2 is used by historical implementations for grant/deleted records; + // other high unsigned ids are special directory records. The shared + // writer does not reproduce those structures, so refuse rather than drop them. + if (parent == DirIdGrantOrDeleted || parent >= HighestSpecialDirectoryId) + throw new NotSupportedException( + "NWFS386 mutation refused special/deleted/trustee directory records that the plain writer cannot preserve."); + + // DOS namespace is zero; any other namespace may carry a parallel name + // or metadata record the plain writer cannot round-trip. + if (entry[10] != 0 || entry[9] != 0) + throw new NotSupportedException( + "NWFS386 mutation is limited to the primary DOS namespace without special directory-entry flags."); + + var attributes = BinaryPrimitives.ReadUInt32LittleEndian(entry[4..]); + if ((attributes & ~(AttributeDirectory | AttributeArchive)) != 0) + throw new NotSupportedException( + "NWFS386 mutation refused file attributes outside the plain writer's preserved subset."); + + if ((attributes & AttributeDirectory) == 0 && + BinaryPrimitives.ReadUInt32LittleEndian(entry[104..]) != 0) + throw new NotSupportedException( + "NWFS386 mutation refused a volume containing salvage/deleted-file records."); + } + + var fatOffset = layout.DataAreaOffset + + (long)(root - layout.FirstSegmentBlock) * FatEntryBytes + + 4; + if (fatOffset < 0 || fatOffset + 4 > image.LongLength) + throw new NotSupportedException("NWFS386 mutation refused a truncated directory FAT chain."); + root = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan((int)fatOffset, 4)); + } + + if (!sawVolumeInfo) + throw new NotSupportedException("NWFS386 mutation requires the root volume-information record."); + + var items = volume.List(); + ValidateFileChains(image, layout, items); + + // Empty directories cannot be represented by the shared writer yet. Refuse + // such a source for rewrite operations so an edit never makes one disappear. + var files = items.Where(static item => !item.IsDirectory).Select(static item => item.Path).ToArray(); + foreach (var directory in items.Where(static item => item.IsDirectory)) + if (!files.Any(file => file.StartsWith(directory.Path + "/", StringComparison.OrdinalIgnoreCase))) + throw new NotSupportedException( + $"NWFS386 mutation refused empty directory '{directory.Path}' because the current writer cannot preserve it."); + } + + private static void ValidateFileChains( + byte[] image, + PlainLayout layout, + IReadOnlyList items) { + foreach (var item in items.Where(static item => !item.IsDirectory)) { + var expectedBlocks = item.Length == 0 + ? 0L + : (item.Length + layout.BlockSize - 1) / layout.BlockSize; + + if (expectedBlocks == 0) { + if (item.FirstBlock != NoBlock) + throw new NotSupportedException( + $"NWFS386 mutation refused empty file '{item.Path}' with an allocated data chain."); + continue; + } + + var block = item.FirstBlock; + for (long expectedIndex = 0; expectedIndex < expectedBlocks; ++expectedIndex) { + if (block == NoBlock) + throw new NotSupportedException( + $"NWFS386 mutation refused truncated/sparse FAT chain for '{item.Path}'."); + + var relativeBlock = (long)block - layout.FirstSegmentBlock; + var fatOffset = layout.DataAreaOffset + relativeBlock * FatEntryBytes; + if (relativeBlock < 0 || fatOffset < 0 || fatOffset + FatEntryBytes > image.LongLength) + throw new NotSupportedException( + $"NWFS386 mutation refused out-of-range FAT chain for '{item.Path}'."); + + var fatIndex = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan((int)fatOffset, 4)); + if (fatIndex != expectedIndex) + throw new NotSupportedException( + $"NWFS386 mutation refused sparse/non-linear FAT indexing for '{item.Path}'."); + + block = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan((int)fatOffset + 4, 4)); + } + + if (block != NoBlock) + throw new NotSupportedException( + $"NWFS386 mutation refused an overlong FAT chain for '{item.Path}'."); + } + } + + private static bool TryLocateVolume(ReadOnlySpan image, out PlainLayout layout) { + layout = default; + var hotfix = FindHotfix(image); + if (hotfix < 0 || hotfix + 28 > image.Length) + return false; + + var redirectionSectors = + BinaryPrimitives.ReadUInt32LittleEndian(image.Slice(checked((int)hotfix + 24), 4)); + var volumeArea = hotfix + (long)redirectionSectors * SectorSize; + if (volumeArea < 0 || volumeArea + 32 + 60 > image.Length) + return false; + if (!image.Slice((int)volumeArea, 16).SequenceEqual("NetWare Volumes\0"u8)) + return false; + + var count = BinaryPrimitives.ReadUInt32LittleEndian(image.Slice((int)volumeArea + 16, 4)); + if (count != 1) + return false; + + var entry = image.Slice((int)volumeArea + 32, 60); + + // VOLUME_TABLE_ENTRY.LastVolumeSegment == 0 is the single-segment profile. + if (BinaryPrimitives.ReadUInt32LittleEndian(entry[16..]) != 0) + return false; + + var firstSegmentBlock = BinaryPrimitives.ReadUInt32LittleEndian(entry[36..]); + var blockValue = BinaryPrimitives.ReadUInt32LittleEndian(entry[44..]); + if (blockValue == 0 || (256u * 1024u) % blockValue != 0) + return false; + var blockSize = checked((int)(256u * 1024u / blockValue)); + if (!SupportedBlockSizes.Contains(blockSize)) + return false; + + var rootDirectoryBlock = BinaryPrimitives.ReadUInt32LittleEndian(entry[48..]); + var dataAreaOffset = volumeArea + VolumeAreaBytes; + if (dataAreaOffset < 0 || dataAreaOffset >= image.Length) + return false; + + layout = new PlainLayout(dataAreaOffset, blockSize, firstSegmentBlock, rootDirectoryBlock); + return true; + } + + private static long FindHotfix(ReadOnlySpan image) { + const int partitionTableOffset = 446; + const byte netWare386PartitionType = 0x65; + const long hotfixOffsetInPartition = 0x4000; + + if (image.Length >= 512 && image[510] == 0x55 && image[511] == 0xAA) { + for (var i = 0; i < 4; ++i) { + var entry = image.Slice(partitionTableOffset + i * 16, 16); + if (entry[4] != netWare386PartitionType) continue; + + var partitionStart = + (long)BinaryPrimitives.ReadUInt32LittleEndian(entry[8..]) * SectorSize; + var at = partitionStart + hotfixOffsetInPartition; + if (at >= 0 && at + 8 <= image.Length && + image.Slice((int)at, 8).SequenceEqual("HOTFIX00"u8)) + return at; + } + } + + return hotfixOffsetInPartition + 8 <= image.Length && + image.Slice((int)hotfixOffsetInPartition, 8).SequenceEqual("HOTFIX00"u8) + ? hotfixOffsetInPartition + : -1; + } + + private static string NormalizePath(string path) + => (path ?? string.Empty).Replace('\\', '/').Trim('/'); + + private readonly record struct OpenedVolume(byte[] Image, NwfsReader Volume); + private readonly record struct PlainLayout( + long DataAreaOffset, + int BlockSize, + uint FirstSegmentBlock, + uint RootDirectoryBlock); } From 51001bbe941b67cdf2cf462d30e155deef68b226 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:10:32 +0200 Subject: [PATCH 090/165] Replace bogus NWFS386 magic tests --- .../Nwfs386/Nwfs386DetectionTests.cs | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/Compression.Tests/Nwfs386/Nwfs386DetectionTests.cs b/Compression.Tests/Nwfs386/Nwfs386DetectionTests.cs index 4a4e0f0e7..79250ed9b 100644 --- a/Compression.Tests/Nwfs386/Nwfs386DetectionTests.cs +++ b/Compression.Tests/Nwfs386/Nwfs386DetectionTests.cs @@ -1,23 +1,32 @@ #pragma warning disable CS1591 using Compression.Lib; +using FileSystem.Nwfs; namespace Compression.Tests.Nwfs386; [TestFixture] public class Nwfs386DetectionTests { - private static byte[] BuildMinimal() { + [Test, Category("ErrorHandling")] + public void Detector_DoesNotTreatArbitraryNetWPrefixAsNwfs386() { var image = new byte[4096]; - // "NetW" ASCII at offset 0. - image[0] = 0x4E; image[1] = 0x65; image[2] = 0x74; image[3] = 0x57; - return image; + "NetW"u8.CopyTo(image); + + var fmt = FormatDetector.DetectByMagic(image); + + Assert.That(fmt.ToString(), Is.Not.EqualTo("Nwfs386").IgnoreCase, + "The old four-byte NetW prefix had no authoritative NWFS386 basis and must not claim the format."); } [Test, Category("HappyPath")] - public void Detector_IdentifiesNwfs386_ByMagic() { - var image = BuildMinimal(); - var fmt = FormatDetector.DetectByMagic(image.AsSpan(0, 512)); - Assert.That(fmt.ToString(), Is.EqualTo("Nwfs386").IgnoreCase, - $"FormatDetector must recognise NWFS386 via 'NetW' at offset 0. Got: {fmt}"); + public void HotfixMagic_IsOwnedBySharedNwfsDetector_NotDuplicatedByAlias() { + var writer = new NwfsWriter(); + writer.AddFile("HELLO.TXT", "hello"u8.ToArray()); + var image = writer.Build(); + + var fmt = FormatDetector.DetectByMagic(image); + + Assert.That(fmt.ToString(), Is.EqualTo("Nwfs").IgnoreCase, + "Nwfs386 is an extension-routed compatibility id; the shared NWFS descriptor owns HOTFIX00 magic routing."); } } From b4a3daafdef5d986607e47968d94f3b479b3d81f Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:10:41 +0200 Subject: [PATCH 091/165] Promote eCryptfs to password-aware WORM --- .../EcryptfsFormatDescriptor.cs | 120 +++++++++++++----- 1 file changed, 91 insertions(+), 29 deletions(-) diff --git a/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs b/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs index 1e12e1e77..3cb42e901 100644 --- a/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs +++ b/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs @@ -5,55 +5,117 @@ namespace FileSystem.Ecryptfs; /// -/// Read-only descriptor for eCryptfs per-file encryption containers. -/// eCryptfs (Linux) stacks on top of any underlying FS and stores each -/// encrypted file with a 4-byte big-endian marker 0x3C81B7F5 at -/// offset 0 followed by an 8-byte decrypted size, 4-byte flags, and -/// 4-byte extent-size hint. Decryption requires the user's passphrase + -/// EFEK packets — out of scope. The encrypted payload is surfaced as a -/// single opaque entry along with the parsed header metadata. +/// Descriptor for one eCryptfs lower file. eCryptfs is a stacked filesystem: +/// every lower file carries its own encrypted upper-file payload, size, extent +/// geometry and authentication-token packet set. /// -/// References: -/// -/// https://docs.kernel.org/filesystems/ecryptfs.html — Linux kernel eCryptfs documentation -/// https://github.com/torvalds/linux/tree/master/fs/ecryptfs — mainline implementation (ecryptfs_kernel.h defines the file-header marker + packet layout) -/// https://en.wikipedia.org/wiki/ECryptfs — Wikipedia overview -/// +/// +/// The implementation supports the ordinary passphrase path using the Linux +/// eCryptfs AES-128/192/256 packet conventions. Private-key authentication and +/// xattr-only metadata are deliberately rejected because their required key/xattr +/// material is not contained in a standalone stream. +/// /// -public sealed class EcryptfsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { +public sealed class EcryptfsFormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IArchiveCreatable, + IArchiveShrinkable, + IWipeEmpty, + IArchivePurgeable { + public string Id => "Ecryptfs"; public string DisplayName => "eCryptfs"; public FormatCategory Category => FormatCategory.Archive; public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract; + FormatCapabilities.CanList + | FormatCapabilities.CanExtract + | FormatCapabilities.CanCreate + | FormatCapabilities.SupportsPassword; public string DefaultExtension => ".ecryptfs"; public IReadOnlyList Extensions => [".ecryptfs"]; public IReadOnlyList CompoundExtensions => []; - public IReadOnlyList MagicSignatures => [ - // eCryptfs marker: 0x3C81B7F5 big-endian at file offset 0. - new([0x3C, 0x81, 0xB7, 0xF5], Offset: 0, Confidence: 0.95), + /// + /// eCryptfs has no fixed byte magic: its marker is two random 32-bit words + /// whose XOR equals 0x3C81B7F5. Static signature matching cannot express that + /// relation, so detection falls back to the extension and the reader performs + /// the authoritative relational check. + /// + public IReadOnlyList MagicSignatures => []; + + public IReadOnlyList Methods => [ + new("aes128", "AES-128-CBC"), + new("aes192", "AES-192-CBC"), + new("aes256", "AES-256-CBC"), ]; - public IReadOnlyList Methods => [new("stored", "Stored")]; public string? TarCompressionFormatId => null; public AlgorithmFamily Family => AlgorithmFamily.Archive; - public string Description => "eCryptfs file-level encryption container — header surface + opaque ciphertext."; + public string Description => "Linux eCryptfs per-file lower-file encryption container."; public List List(Stream stream, string? password) { - var r = new EcryptfsReader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo( - i, e.Name, e.Size, e.Size, "Stored", e.IsDirectory, false, null)).ToList(); + using var reader = new EcryptfsReader(stream); + if (reader.DecryptedSize == 0) return []; + return [new ArchiveEntryInfo( + 0, + "content.bin", + checked((long)reader.DecryptedSize), + reader.CanonicalLength - reader.MetadataSize, + reader.CipherDescription, + IsDirectory: false, + IsEncrypted: true, + LastModified: null + )]; } public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new EcryptfsReader(stream); - foreach (var e in r.Entries) { - if (e.IsDirectory) continue; - if (files != null && !MatchesFilter(e.Name, files)) continue; - WriteFile(outputDir, e.Name, r.Extract(e)); - } + using var reader = new EcryptfsReader(stream); + if (reader.DecryptedSize == 0) return; + if (files != null && !MatchesFilter("content.bin", files)) return; + if (password is null) + throw new ArgumentException("A passphrase is required to extract an eCryptfs lower file.", nameof(password)); + WriteFile(outputDir, "content.bin", reader.ExtractContent(password)); } + + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + ArgumentNullException.ThrowIfNull(options); + if (options.Password is null) + throw new ArgumentException("A passphrase is required to create an eCryptfs lower file.", nameof(options)); + + var files = inputs.Where(i => !i.IsDirectory).ToList(); + if (inputs.Any(i => i.IsDirectory)) + throw new NotSupportedException("An eCryptfs lower file represents one regular file; directories are not embedded in it."); + if (files.Count > 1) + throw new NotSupportedException("An eCryptfs lower file can contain exactly one logical file payload."); + + var content = files.Count == 0 ? Array.Empty() : files[0].ReadContent(); + var method = options.EncryptionMethod ?? options.MethodName; + EcryptfsCodec.Create(output, content, options.Password, EcryptfsCodec.ResolveKeySize(method)); + } + + /// + /// Clears metadata padding and bytes beyond the canonical encrypted extent set. + /// Ciphertext inside the final live extent is never touched: plaintext tail + /// zeroing would require the passphrase, which the wipe interface intentionally + /// does not carry. + /// + public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) + => EcryptfsCodec.WipeUnusedSpace(image); + + /// + /// Drops any bytes after the encrypted extents implied by the header. This is + /// the only meaningful compact/shrink operation for a single eCryptfs lower file. + /// + public void Shrink(Stream input, Stream output) => EcryptfsCodec.Shrink(input, output); + + /// + /// Removes the logical payload without requiring the FEK: set plaintext length + /// to zero and truncate all ciphertext extents while retaining valid key metadata. + /// + public void Purge(Stream archive) => EcryptfsCodec.Purge(archive); } From 76fa6d1fae4efd997d3edb0874102170788a903d Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:10:58 +0200 Subject: [PATCH 092/165] * refresh TI-TXT support matrix capabilities --- Hawkynt.FileFormats.FileSystems/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index d5725e500..1c11dd4cc 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -72,9 +72,9 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [QCOW2](https://en.wikipedia.org/wiki/Qcow) | `Qcow2` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | [`qemu-img check`](https://www.qemu.org/docs/master/tools/qemu-img.html) / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [QCOW2 specification](https://www.qemu.org/docs/master/interop/qcow2.html) | | [T64](https://en.wikipedia.org/wiki/Commodore_64) | `T64` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 tape archive | [T64 (VICE)](https://vice-emu.sourceforge.io/vice_17.html) | | [TAP](https://en.wikipedia.org/wiki/ZX_Spectrum) | `Tap` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | Sinclair / Commodore tape image | [TAP (World of Spectrum)](https://worldofspectrum.net/faq/reference/formats.htm) | -| [TI-TXT (MSP430)](https://en.wikipedia.org/wiki/Texas_Instruments) | `TiTxt` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | TI-TXT firmware dump; the writer takes one payload and the `@address` from `metadata.ini` | [TI-TXT](https://www.ti.com/lit/pdf/slau131) | +| [TI-TXT (MSP430)](https://en.wikipedia.org/wiki/Texas_Instruments) | `TiTxt` | R/W | ✅ | ✅ rebuild | — | — | — | ✅ | own reader + struct-parity tests | TI-TXT sparse firmware dump; edits and canonical rebuild preserve omitted address holes | [TI-TXT](https://www.ti.com/lit/pdf/slau131) | | [U-Boot uImage](https://en.wikipedia.org/wiki/Das_U-Boot) | `UImage` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | U-Boot legacy image header + payload; the writer takes one payload and the header fields from `metadata.ini`, and computes both CRCs | [U-Boot image format](https://docs.u-boot.org/en/latest/usage/fit/index.html) | -| [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | `UefiFv` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Firmware volume / FFS | [UEFI specification](https://uefi.org/specifications) | +| [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | `UefiFv` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Firmware volume / FFS | [UEFI specification](https://uefi.org/specifications/) | | [VDI](https://en.wikipedia.org/wiki/VirtualBox#Virtual_disk_image) | `Vdi` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [VirtualBox storage](https://www.virtualbox.org/manual/ch05.html) | | [VHD](https://en.wikipedia.org/wiki/VHD_(file_format)) | `Vhd` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Standalone fixed and dynamic; differencing requires a parent chain and is rejected; edits go to the inner filesystem | [Microsoft VHD](https://learn.microsoft.com/windows-server/virtualization/hyper-v/manage/manage-hyper-v-virtual-hard-disks) | | [VHDX](https://en.wikipedia.org/wiki/VHD_(file_format)#VHDX) | `Vhdx` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [MS-VHDX](https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-vhdx/) | @@ -86,9 +86,9 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | --- | --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | --- | --- | --- | | [DoubleSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DoubleSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` driver under QEMU mounts, reads and writes; real driver CVFs read byte-exact | MSDBL6.0 CVF; stored clusters | [DoubleSpace](https://en.wikipedia.org/wiki/DriveSpace) | | [DriveSpace 3 CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace3` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | `dmsdos` mounts the genuine `GenuineDvr3Writer` output and reads byte-exact | Win95 DVR3 CVF; a Win95 guest oracle and the HiPack codec are still open | [DriveSpace 3 notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/FileSystems/FileSystem.DriveSpace3/FORMAT-NOTES.md) | -| [DriveSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` driver under QEMU mounts, reads and writes; real driver CVFs read byte-exact | MSDSP6.0 CVF; stored clusters | [DriveSpace](https://en.wikipedia.org/wiki/DriveSpace) | +| [DriveSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` mounts the DoubleSpace / DriveSpace CVFs; the `dmsdos` driver mounts the DriveSpace 3 and Stacker volumes. | MSDSP6.0 CVF; stored clusters | [DriveSpace](https://en.wikipedia.org/wiki/DriveSpace) | | [exFAT](https://en.wikipedia.org/wiki/ExFAT) | `ExFat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `exfat` mount r/w + [`fsck.exfat`](https://github.com/exfatprogs/exfatprogs) | — | [exFAT specification](https://learn.microsoft.com/windows/win32/fileio/exfat-specification) | -| [FAT Filesystem Image](https://en.wikipedia.org/wiki/File_Allocation_Table) | `Fat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount r/w + [`fsck.fat`](https://github.com/dosfstools/dosfstools) + [`mtools`](https://www.gnu.org/software/mtools/) + reverse [`mkfs.vfat`](https://github.com/dosfstools/dosfstools) | FAT12/16/32 with long filenames; in-place `FatModifier` | [FAT specification](https://download.microsoft.com/download/1/6/1/161ba512-40e2-4cc9-843a-923143f3456c/fatgen103.doc) | +| [FAT Filesystem Image](https://en.wikipedia.org/wiki/File_Allocation_Table) | `Fat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount + [`fsck.fat`](https://github.com/dosfstools/dosfstools) + [`mtools`](https://www.gnu.org/software/mtools/) read the written volume | FAT12/16/32 with long filenames; in-place `FatModifier` | [FAT specification](https://download.microsoft.com/download/1/6/1/161ba512-40e2-4cc9-843a-923143f3456c/fatgen103.doc) | | [FAT+ Filesystem Image (large-file extension)](https://en.wikipedia.org/wiki/File_Allocation_Table#FAT+) | `FatPlus` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount + [`fsck.fat`](https://github.com/dosfstools/dosfstools) read the written volume | FAT+ large-file extension | [FAT+ draft](https://www.fdos.org/kernel/fatplus.txt) | | [FATX (Xbox)](https://en.wikipedia.org/wiki/FATX) | `Fatx` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Xbox FATX | [FATX (xboxdevwiki)](https://xboxdevwiki.net/FATX) | | [HPFS](https://en.wikipedia.org/wiki/High_Performance_File_System) | `Hpfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | OS/2 HPFS | [HPFS layout (OS/2 Museum)](https://www.os2museum.com/wp/the-hpfs-disk-layout/) | @@ -114,7 +114,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [GFS (Sistina/Red Hat, original)](https://en.wikipedia.org/wiki/GFS2) | `Gfs1` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; the Linux `gfs2` driver rejects GFS1 | Sistina GFS | [GFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | | [GFS2 (Global File System 2)](https://en.wikipedia.org/wiki/GFS2) | `Gfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.gfs2 -n`](https://pagure.io/gfs2-utils) clean on populated create and after add/replace/remove; reverse `mkfs.gfs2` reader coverage | Standalone `lock_nolock`; stuffed + indirect-tree files; edits rebuild while preserving image-size floor and lock table | [GFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | | [HAMMER (DragonFly BSD)](https://en.wikipedia.org/wiki/HAMMER_(file_system)) | `Hammer` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | DragonFly kernel mount r/w under QEMU + [`hammer show` / `hammer checkmap`](https://www.dragonflybsd.org/hammer/) | Single volume, about 1 GB minimum; edits rebuild the volume | [HAMMER](https://www.dragonflybsd.org/hammer/) | -| [HAMMER2 (DragonFly BSD)](https://en.wikipedia.org/wiki/HAMMER2) | `Hammer2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | DragonFly kernel mount r/w under QEMU | — | [HAMMER2](https://www.dragonflybsd.org/hammer/) | +| [HAMMER2 (DragonFly BSD)](https://en.wikipedia.org/wiki/HAMMER2) | `Hammer2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | DragonFly kernel mount r/w under QEMU | — | [HAMMER2](https://www.dragonflybsd.org/hammer2/) | | [HTFS (SCO High Throughput File System)](https://en.wikipedia.org/wiki/High_Throughput_File_System) | `Htfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | SCO HTFS | [HTFS](https://en.wikipedia.org/wiki/High_Throughput_File_System) | | [JFS](https://en.wikipedia.org/wiki/JFS_(file_system)) | `Jfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `jfs` mount + [`fsck.jfs`](http://jfs.sourceforge.net/) | JFS2 (Linux / AIX) | [JFS project](http://jfs.sourceforge.net/) | | [JFS1 (OS/2 original IBM JFS)](https://en.wikipedia.org/wiki/JFS_(file_system)) | `Jfs1` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; the Linux `jfs` driver only knows JFS2 | OS/2 JFS1 | [JFS project](http://jfs.sourceforge.net/) | @@ -166,7 +166,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [JFFS2](https://en.wikipedia.org/wiki/JFFS2) | `Jffs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`mkfs.jffs2`](http://git.infradead.org/mtd-utils.git) image read byte-exact | Log-structured | [JFFS2](https://www.kernel.org/doc/html/latest/filesystems/jffs2.html) | | [LittleFS](https://github.com/littlefs-project/littlefs) | `LittleFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [littlefs](https://github.com/littlefs-project/littlefs/blob/master/SPEC.md) | | [ROMFS](https://en.wikipedia.org/wiki/Romfs) | `RomFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `romfs` mount + [7-Zip](https://www.7-zip.org/) read the written image | Edits rebuild the image | [romfs](https://www.kernel.org/doc/html/latest/filesystems/romfs.html) | -| [SmartFS](https://nuttx.apache.org/docs/latest/components/filesystem/smartfs.html) | `SmartFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | NuttX; flat root; edits rebuild the volume | [NuttX SmartFS](https://github.com/apache/nuttx/tree/master/fs/smartfs) | +| [SmartFS](https://nuttx.apache.org/docs/latest/components/filesystem/smartfs.html) | `SmartFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [NuttX SmartFS](https://github.com/apache/nuttx/tree/master/fs/smartfs) | | [SquashFS](https://en.wikipedia.org/wiki/SquashFS) | `SquashFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `squashfs` mount + [`unsquashfs`](https://github.com/plougher/squashfs-tools) | Edits rebuild the image | [SquashFS](https://docs.kernel.org/filesystems/squashfs.html) | | [UBIFS](https://en.wikipedia.org/wiki/UBIFS) | `Ubifs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Log-structured; edits rebuild the image | [UBIFS](https://www.kernel.org/doc/html/latest/filesystems/ubifs.html) | | [YAFFS2](https://en.wikipedia.org/wiki/YAFFS) | `Yaffs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [YAFFS](https://yaffs.net/) | @@ -197,7 +197,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [SCL (ZX Spectrum)](https://en.wikipedia.org/wiki/TR-DOS) | `ZxScl` | R/W | ✅ | ✅ moving | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | SCL, `SINCLAIR` magic + checksum | [SCL](https://sinclair.wiki.zxnet.co.uk/wiki/TR-DOS) | | [Sharp X68000 Human68k](https://en.wikipedia.org/wiki/Human68k) | `Human68k` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Sharp X68000 | [Human68k](https://en.wikipedia.org/wiki/Human68k) | | [TI-99/4A DSR](https://en.wikipedia.org/wiki/TI-99/4A) | `Ti99` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [TI-99/4A disk format](https://www.unige.ch/medecine/nouspikel/ti99/disks.htm) | -| [TR-DOS](https://en.wikipedia.org/wiki/TR-DOS) | `TrDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | ZX Spectrum TRD | [TR-DOS](https://sinclair.wiki.zxnet.co.uk/wiki/TR-DOS) | +| [TR-DOS](https://en.wikipedia.org/wiki/TR-DOS) | `TrDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | ZX Spectrum TRD | [TR-DOS](https://en.wikipedia.org/wiki/TR-DOS) | | [TRSDOS / LDOS](https://en.wikipedia.org/wiki/TRSDOS) | `Trsdos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [TRSDOS](https://en.wikipedia.org/wiki/TRSDOS) | | [VDFS](https://en.wikipedia.org/wiki/Gothic_(series)) | `Vdfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Gothic-engine VDFS | [VDFS](https://github.com/PhoenixTales/vdfs) | @@ -219,7 +219,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [CephFS / RADOS](https://en.wikipedia.org/wiki/Ceph_(software)) | `CephFs` | R | — | — | — | — | — | — | detection of the on-disk signature | RADOS objects only | [Ceph](https://docs.ceph.com/) | | [Dell EMC Isilon OneFS](https://en.wikipedia.org/wiki/OneFS_distributed_file_system) | `OneFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Isilon OneFS | [OneFS](https://www.dell.com/en-us/dt/storage/powerscale.htm) | | [eCryptfs](https://en.wikipedia.org/wiki/ECryptfs) | `Ecryptfs` | R | — | — | — | — | — | — | detection of the on-disk signature | Payload is encrypted; plaintext needs the FEK | [eCryptfs](https://www.kernel.org/doc/html/latest/filesystems/ecryptfs.html) | -| [GlusterFS](https://en.wikipedia.org/wiki/Gluster) | `GlusterFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Brick metadata only | [GlusterFS](https://docs.gluster.org/) | +| [GlusterFS](https://en.wikipedia.org/wiki/GlusterFS) | `GlusterFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Brick metadata only | [GlusterFS](https://docs.gluster.org/) | | [IBM Spectrum Scale / GPFS](https://en.wikipedia.org/wiki/GPFS) | `Gpfs` | R | — | — | — | — | — | — | detection of the on-disk signature | IBM Spectrum Scale | [GPFS](https://www.ibm.com/docs/en/storage-scale) | | [JuiceFS](https://juicefs.com/) | `JuiceFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Chunk objects only | [JuiceFS](https://juicefs.com/docs/community/introduction/) | | [Lustre](https://en.wikipedia.org/wiki/Lustre_(file_system)) | `Lustre` | R | — | — | — | — | — | — | detection of the on-disk signature | OST / MDT objects only | [Lustre](https://www.lustre.org/) | From 1a1022614c7a75a8105f34ae6a44d3c21ab08e36 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:10:58 +0200 Subject: [PATCH 093/165] Add NWFS386 R/W and maintenance coverage --- .../Nwfs386/Nwfs386StubBehaviorTests.cs | 207 +++++++++++++----- 1 file changed, 158 insertions(+), 49 deletions(-) diff --git a/Compression.Tests/Nwfs386/Nwfs386StubBehaviorTests.cs b/Compression.Tests/Nwfs386/Nwfs386StubBehaviorTests.cs index 30385909d..f4ef6e567 100644 --- a/Compression.Tests/Nwfs386/Nwfs386StubBehaviorTests.cs +++ b/Compression.Tests/Nwfs386/Nwfs386StubBehaviorTests.cs @@ -1,64 +1,173 @@ #pragma warning disable CS1591 +using System.Buffers.Binary; using Compression.Registry; +using FileSystem.Nwfs; using FileSystem.Nwfs386; namespace Compression.Tests.Nwfs386; -/// -/// Pins the stub-tier surface for . Novell -/// NetWare 386 has no public on-disk spec — these tests prevent silent capability -/// creep (CanCreate/CanModify) and stop the opaque-blob entry shape from drifting. -/// [TestFixture] -public class Nwfs386StubBehaviorTests { +public class Nwfs386BehaviorTests { - private static byte[] BuildMagicOnly() { - var image = new byte[4096]; - // "NetW" - image[0] = 0x4E; image[1] = 0x65; image[2] = 0x74; image[3] = 0x57; + private static readonly byte[] Hello = "hello nwfs386"u8.ToArray(); + private static readonly byte[] Readme = Enumerable.Range(0, 5000).Select(i => (byte)(i * 31)).ToArray(); + + private static MemoryStream CreateImage(int blockSize = 4096) { + var image = new MemoryStream(); + new Nwfs386FormatDescriptor().Create( + image, + [ + ArchiveInputInfo.InMemory("HELLO.TXT", Hello), + ArchiveInputInfo.InMemory("PUBLIC/README.DOC", Readme), + ], + new FormatCreateOptions { + FormatSpecific = new Dictionary(StringComparer.OrdinalIgnoreCase) { + ["BlockSize"] = blockSize.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["VolumeName"] = "SYS", + }, + }); + image.Position = 0; return image; } - [Test, Category("Stub")] - public void Stub_DescriptorHonestlyAdvertisesCapabilities_AndOpaqueEntries() { - var d = new Nwfs386FormatDescriptor(); - - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False, - "NWFS386 is stub-tier (no public spec) — must not advertise CanCreate."); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False, - "NWFS386 is stub-tier (no public spec) — must not advertise CanModify."); - - var image = BuildMagicOnly(); - using var ms = new MemoryStream(image, writable: false); - var entries = d.List(ms, null); - - var names = entries.Select(e => e.Name).ToList(); - Assert.That(names, Is.EquivalentTo(new[] { "FULL.nwfs386", "metadata.ini" }), - "NWFS386 minimal-image surface must be exactly the documented opaque pair: FULL.nwfs386 + metadata.ini."); - - var outDir = Path.Combine(Path.GetTempPath(), "Nwfs386Stub_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(outDir); - try { - using var ms2 = new MemoryStream(image, writable: false); - d.Extract(ms2, outDir, password: null, files: null); - var fullPath = Path.Combine(outDir, "FULL.nwfs386"); - Assert.That(File.Exists(fullPath), Is.True, "Extract must produce FULL.nwfs386."); - var roundTrip = File.ReadAllBytes(fullPath); - Assert.That(roundTrip, Is.EqualTo(image), - "FULL.nwfs386 must round-trip the magic-only input byte-for-byte."); - } finally { - Directory.Delete(outDir, recursive: true); - } + [Test, Category("HappyPath")] + public void Descriptor_AdvertisesTheBackedRwSurface() { + var descriptor = new Nwfs386FormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf(), + "NWFS386 must not claim wipe until all free/salvage/suballocation regions can be proven safely."); + }); + } + + [Test, Category("HappyPath")] + public void Create_ListAndExtract_RoundTripFilesAndDirectories() { + using var image = CreateImage(); + var descriptor = new Nwfs386FormatDescriptor(); + + var entries = descriptor.List(image, null); + + Assert.That(entries.Select(entry => entry.Name), Does.Contain("HELLO.TXT")); + Assert.That(entries.Select(entry => entry.Name), Does.Contain("PUBLIC")); + Assert.That(entries.Select(entry => entry.Name), Does.Contain("PUBLIC/README.DOC")); + + image.Position = 0; + using var opened = descriptor.OpenEntry(image, "PUBLIC/README.DOC", null); + using var copied = new MemoryStream(); + opened.CopyTo(copied); + Assert.That(copied.ToArray(), Is.EqualTo(Readme)); + } + + [Test, Category("HappyPath")] + public void AddReplaceRemove_PreservesGeometryAndLiveBytes() { + using var image = CreateImage(8192); + var descriptor = new Nwfs386FormatDescriptor(); + var replacement = "replacement"u8.ToArray(); + var added = "new file"u8.ToArray(); + + descriptor.Add(image, [ + ArchiveInputInfo.InMemory("HELLO.TXT", replacement), + ArchiveInputInfo.InMemory("NEW.BIN", added), + ]); + descriptor.Remove(image, ["PUBLIC/README.DOC"]); + + var volume = NwfsReader.TryOpen(image.ToArray()); + Assert.That(volume, Is.Not.Null); + Assert.Multiple(() => { + Assert.That(volume!.BlockSize, Is.EqualTo(8192)); + Assert.That(volume.VolumeName, Is.EqualTo("SYS")); + Assert.That(volume.ReadFile("HELLO.TXT"), Is.EqualTo(replacement)); + Assert.That(volume.ReadFile("NEW.BIN"), Is.EqualTo(added)); + Assert.That(volume.ReadFile("PUBLIC/README.DOC"), Is.Null); + }); + } + + [Test, Category("HappyPath")] + public void Purge_LeavesAValidEmptyVolume() { + using var image = CreateImage(); + var descriptor = new Nwfs386FormatDescriptor(); + + ((IArchivePurgeable)descriptor).Purge(image); + + image.Position = 0; + Assert.That(descriptor.List(image, null).Where(entry => !entry.IsDirectory), Is.Empty); + Assert.That(NwfsReader.TryOpen(image.ToArray()), Is.Not.Null); + } + + [Test, Category("HappyPath")] + public void DefragShrinkAndLayout_PreserveContent() { + using var image = CreateImage(4096); + var descriptor = new Nwfs386FormatDescriptor(); + + descriptor.Defragment(image); + var defragged = NwfsReader.TryOpen(image.ToArray())!; + Assert.That(defragged.ReadFile("PUBLIC/README.DOC"), Is.EqualTo(Readme)); + + var padded = image.ToArray().Concat(new byte[128 * 1024]).ToArray(); + using var paddedStream = new MemoryStream(padded, writable: false); + using var shrunk = new MemoryStream(); + descriptor.Shrink(paddedStream, shrunk); + Assert.That(shrunk.Length, Is.LessThan(padded.LongLength)); + Assert.That(NwfsReader.TryOpen(shrunk.ToArray())!.ReadFile("HELLO.TXT"), Is.EqualTo(Hello)); + + image.Position = 0; + using var relaid = new MemoryStream(); + descriptor.RebuildStreaming(image, relaid, new LayoutRebuildOptions { UnitSize = 8192 }); + var layoutVolume = NwfsReader.TryOpen(relaid.ToArray())!; + Assert.Multiple(() => { + Assert.That(layoutVolume.BlockSize, Is.EqualTo(8192)); + Assert.That(layoutVolume.ReadFile("HELLO.TXT"), Is.EqualTo(Hello)); + Assert.That(layoutVolume.ReadFile("PUBLIC/README.DOC"), Is.EqualTo(Readme)); + }); + } + + [Test, Category("HappyPath")] + public void AnalyzeLayout_ReportsCurrentAndCandidateBlockSize() { + using var image = CreateImage(16384); + var descriptor = new Nwfs386FormatDescriptor(); + + var analysis = descriptor.AnalyzeLayout(image); + + Assert.That(analysis.CurrentUnitSize, Is.EqualTo(16384)); + Assert.That(analysis.OptimalUnitSize, Is.AnyOf(1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144)); + Assert.That(analysis.OptimalSlackBytes, Is.LessThanOrEqualTo(analysis.CurrentSlackBytes)); + } + + [Test, Category("ErrorHandling")] + public void Mutation_RejectsCompressionProfileBeforeChangingSource() { + using var image = CreateImage(); + var bytes = image.ToArray(); + var volumeArea = FindVolumeArea(bytes); + var entry = bytes.AsSpan(volumeArea + 32, 60); + var blockValue = BinaryPrimitives.ReadUInt32LittleEndian(entry[44..]); + var blockSize = checked((int)(256u * 1024u / blockValue)); + var firstDirectory = BinaryPrimitives.ReadUInt32LittleEndian(entry[48..]); + var dataArea = volumeArea + 4 * 16384; + var root = checked(dataArea + (int)firstDirectory * blockSize); + bytes[root + 23] |= 0x04; // FILE_COMPRESSION_ON in the ROOT volume flags. + + using var advanced = new MemoryStream(bytes, writable: true); + var before = advanced.ToArray(); + var descriptor = new Nwfs386FormatDescriptor(); + + Assert.Throws(() => descriptor.Add( + advanced, + [ArchiveInputInfo.InMemory("NEW.BIN", "nope"u8.ToArray())])); + Assert.That(advanced.ToArray(), Is.EqualTo(before), "Rejected mutation must leave the source byte-identical."); } - [Test, Category("Stub")] - public void Stub_DoesNotAdvertiseWriteCapability() { - var d = new Nwfs386FormatDescriptor(); - var description = d.Description.ToLowerInvariant(); - Assert.That( - description.Contains("stub") || description.Contains("opaque") - || description.Contains("skeleton") || description.Contains("detection"), - Is.True, - $"NWFS386 Description must honestly flag its stub/detection-only/opaque status. Got: '{d.Description}'."); + private static int FindVolumeArea(byte[] image) { + var magic = "NetWare Volumes\0"u8; + for (var offset = 0; offset + magic.Length <= image.Length; offset += 512) + if (image.AsSpan(offset, magic.Length).SequenceEqual(magic)) + return offset; + throw new InvalidOperationException("Test image contains no NWFS volume table."); } } From d336912d79c0be5b646c62e98fe0e2a8e2d49ce7 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:11:15 +0200 Subject: [PATCH 094/165] Test real eCryptfs header semantics --- .../Ecryptfs/EcryptfsDetectionTests.cs | 97 ++++++++++++------- 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs b/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs index 28a774719..4aa01d4ca 100644 --- a/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs +++ b/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs @@ -1,55 +1,78 @@ -using System.Buffers.Binary; using Compression.Registry; +using FileSystem.Ecryptfs; namespace Compression.Tests.Ecryptfs; [TestFixture] public class EcryptfsDetectionTests { - // Build a minimal eCryptfs file: marker 0x3C81B7F5 + 8-byte BE decrypted size + 4-byte flags + 4-byte extent-size, - // padded out to one extent so the reader can carve the opaque ciphertext blob. - private static byte[] BuildMinimalFile(ulong decryptedSize = 4096, uint flags = 0, uint extentSize = 4096, int cipherLen = 4096) { - var image = new byte[Math.Max((int)extentSize, 64) + cipherLen]; - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(0, 4), 0x3C81B7F5u); - BinaryPrimitives.WriteUInt64BigEndian(image.AsSpan(4, 8), decryptedSize); - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(12, 4), flags); - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(16, 4), extentSize); - for (var i = 0; i < cipherLen; i++) image[Math.Max((int)extentSize, 64) + i] = (byte)((i * 7) ^ 0xA5); - return image; + private static byte[] CreateImage(byte[] content, string password = "foo", string? encryptionMethod = null) { + var descriptor = new EcryptfsFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create( + image, + [ArchiveInputInfo.InMemory("payload.bin", content)], + new FormatCreateOptions { Password = password, EncryptionMethod = encryptionMethod }); + return image.ToArray(); } [Test, Category("HappyPath")] - public void Descriptor_Properties_AndMagic() { - var d = new FileSystem.Ecryptfs.EcryptfsFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("Ecryptfs")); - Assert.That(d.DisplayName, Is.EqualTo("eCryptfs")); - Assert.That(d.Extensions, Does.Contain(".ecryptfs")); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(1)); - Assert.That(d.MagicSignatures[0].Offset, Is.EqualTo(0)); - Assert.That(d.MagicSignatures[0].Bytes, Is.EqualTo(new byte[] { 0x3C, 0x81, 0xB7, 0xF5 })); - Assert.That(d, Is.Not.InstanceOf()); + public void Descriptor_AdvertisesPasswordAwareWorm_WithoutFakeMagic() { + var descriptor = new EcryptfsFormatDescriptor(); + Assert.That(descriptor.Id, Is.EqualTo("Ecryptfs")); + Assert.That(descriptor.DisplayName, Is.EqualTo("eCryptfs")); + Assert.That(descriptor.Extensions, Does.Contain(".ecryptfs")); + Assert.That(descriptor.MagicSignatures, Is.Empty, + "The eCryptfs marker is a relation between two random words, not fixed bytes."); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.SupportsPassword), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False, + "The generic mutation API carries no passphrase, so plaintext replacement cannot be exposed honestly."); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); } [Test, Category("HappyPath")] - public void Read_MinimalFile_SurfacesHeader() { - using var ms = new MemoryStream(BuildMinimalFile(decryptedSize: 12345, flags: 0x10, extentSize: 4096, cipherLen: 8192)); - var r = new FileSystem.Ecryptfs.EcryptfsReader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.Marker, Is.EqualTo(0x3C81B7F5u)); - Assert.That(r.DecryptedSize, Is.EqualTo(12345ul)); - Assert.That(r.Flags, Is.EqualTo(0x10u)); - Assert.That(r.ExtentSize, Is.EqualTo(4096u)); - - var names = r.Entries.Select(e => e.Name).ToHashSet(); - Assert.That(names, Does.Contain("FULL.ecryptfs")); - Assert.That(names, Does.Contain("metadata.ini")); - Assert.That(names, Does.Contain("ciphertext.bin")); + public void Read_CreatedLowerFile_SurfacesKernelHeaderGeometry() { + var content = Enumerable.Range(0, 12345).Select(i => (byte)(i * 37)).ToArray(); + using var image = new MemoryStream(CreateImage(content), writable: false); + using var reader = new EcryptfsReader(image); + + Assert.That(reader.Marker, Is.EqualTo(0x3C81B7F5u)); + Assert.That(reader.DecryptedSize, Is.EqualTo(12345ul)); + Assert.That(reader.FileVersion, Is.EqualTo(3)); + Assert.That(reader.Flags & 0x00000002u, Is.Not.Zero); + Assert.That(reader.ExtentSize, Is.EqualTo(4096u)); + Assert.That(reader.HeaderExtentCount, Is.EqualTo(2)); + Assert.That(reader.MetadataSize, Is.EqualTo(8192)); + Assert.That(reader.CanonicalLength, Is.EqualTo(8192 + 4 * 4096)); + Assert.That(reader.CipherDescription, Is.EqualTo("AES-128-CBC")); + Assert.That(reader.Entries.Select(e => e.Name), Is.EqualTo(new[] { "content.bin" })); + } + + [Test, Category("Interop")] + public void PassphraseSignature_MatchesEcryptfsUtilsKnownAnswerVector() { + // ecryptfs-utils tests/userspace/verify-passphrase-sig.sh: + // pass="foo", default salt 0011223344556677 => 253ca7e88811d184. + using var image = new MemoryStream(CreateImage([1, 2, 3], password: "foo"), writable: false); + using var reader = new EcryptfsReader(image); + Assert.That(reader.PassphraseSignature, Is.EqualTo("253ca7e88811d184")); + } + + [Test, Category("Sad")] + public void Read_BrokenMarkerRelation_Throws() { + var bytes = CreateImage([1, 2, 3]); + bytes[12] ^= 0x80; + using var image = new MemoryStream(bytes, writable: false); + Assert.Throws(() => _ = new EcryptfsReader(image)); } [Test, Category("Sad")] - public void Read_BadMarker_Throws() { - var img = new byte[64]; - using var ms = new MemoryStream(img); - Assert.Throws(() => _ = new FileSystem.Ecryptfs.EcryptfsReader(ms)); + public void Read_TruncatedCiphertext_Throws() { + var bytes = CreateImage(new byte[4097]); + Array.Resize(ref bytes, bytes.Length - 1); + using var image = new MemoryStream(bytes, writable: false); + Assert.Throws(() => _ = new EcryptfsReader(image)); } } From 10df20d17a8ff3965c506af6bcf3df265d9278e5 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:11:16 +0200 Subject: [PATCH 095/165] Temporarily refresh NRG support matrix --- .github/workflows/nrg-doc-fix.yml | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/nrg-doc-fix.yml diff --git a/.github/workflows/nrg-doc-fix.yml b/.github/workflows/nrg-doc-fix.yml new file mode 100644 index 000000000..867515b8f --- /dev/null +++ b/.github/workflows/nrg-doc-fix.yml @@ -0,0 +1,36 @@ +name: Refresh NRG support matrix + +on: + push: + branches: [fix/nrg-rw-maintenance] + +permissions: + contents: write + +jobs: + refresh: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Refresh NRG support row and remove this one-shot workflow + shell: python + run: | + from pathlib import Path + path = Path("Hawkynt.FileFormats.FileSystems/README.md") + text = path.read_text(encoding="utf-8") + old = "| [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) |" + new = "| [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | ✅ | ✅ rebuild | — | ✅ | — | ✅ | own reader + struct-parity tests | NRG v1/v2 reader; v2 TAO writer; named edits rebuild the embedded ISO | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) |" + if text.count(old) != 1: + raise SystemExit(f"expected exactly one NRG support row, found {text.count(old)}") + path.write_text(text.replace(old, new), encoding="utf-8") + Path(".github/workflows/nrg-doc-fix.yml").unlink() + - name: Commit support matrix refresh + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "Refresh NRG support matrix" + git push origin HEAD:"$GITHUB_REF_NAME" From e809b9ff9af0909f83799fcb4185f2d101da5bae Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:11:33 +0200 Subject: [PATCH 096/165] Cover eCryptfs crypto and maintenance verbs --- .../Ecryptfs/EcryptfsStubBehaviorTests.cs | 152 ++++++++++-------- 1 file changed, 89 insertions(+), 63 deletions(-) diff --git a/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs b/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs index 52dec651c..8cc4651d2 100644 --- a/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs +++ b/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs @@ -1,78 +1,104 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; +using System.Security.Cryptography; using Compression.Registry; using FileSystem.Ecryptfs; namespace Compression.Tests.Ecryptfs; -/// -/// Pins the stub-tier surface for . -/// eCryptfs per-file containers wrap AES-CBC ciphertext extents — without the -/// mount passphrase + EFEK packets the payload is opaque. These tests prevent -/// silent capability creep (CanCreate/CanModify) and stop the opaque-blob entry -/// shape from drifting. -/// [TestFixture] public class EcryptfsStubBehaviorTests { - private static byte[] BuildMagicOnly(int cipherLen = 256) { - // Reader places ciphertext at offset max(ExtentSize, 4096); we set extent=4096. - const int extent = 4096; - var image = new byte[extent + cipherLen]; - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(0, 4), 0x3C81B7F5u); - BinaryPrimitives.WriteUInt64BigEndian(image.AsSpan(4, 8), 8192ul); - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(12, 4), 0u); - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(16, 4), (uint)extent); - for (var i = 0; i < cipherLen; i++) image[extent + i] = (byte)((i * 11) ^ 0xC3); - return image; + private static byte[] CreateImage(byte[] content, string password = "correct horse", string? method = null) { + var descriptor = new EcryptfsFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create( + image, + [ArchiveInputInfo.InMemory("payload.bin", content)], + new FormatCreateOptions { Password = password, EncryptionMethod = method }); + return image.ToArray(); } - [Test, Category("Stub")] - public void Stub_DescriptorHonestlyAdvertisesCapabilities_AndOpaqueEntries() { - var d = new EcryptfsFormatDescriptor(); - - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False, - "eCryptfs is stub-tier (encrypted ciphertext) — must not advertise CanCreate."); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False, - "eCryptfs is stub-tier (encrypted ciphertext) — must not advertise CanModify."); - - var image = BuildMagicOnly(cipherLen: 256); - using var ms = new MemoryStream(image, writable: false); - var entries = d.List(ms, null); - - var names = entries.Select(e => e.Name).ToList(); - Assert.That(names, Is.EquivalentTo(new[] { "FULL.ecryptfs", "metadata.ini", "ciphertext.bin" }), - "eCryptfs minimal-image surface must be exactly the documented opaque triple."); - - var outDir = Path.Combine(Path.GetTempPath(), "EcryptfsStub_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(outDir); - try { - using var ms2 = new MemoryStream(image, writable: false); - d.Extract(ms2, outDir, password: null, files: null); - - var fullPath = Path.Combine(outDir, "FULL.ecryptfs"); - Assert.That(File.Exists(fullPath), Is.True, "Extract must produce FULL.ecryptfs."); - Assert.That(File.ReadAllBytes(fullPath), Is.EqualTo(image), - "FULL.ecryptfs must round-trip the file bytes exactly."); - - var cipherPath = Path.Combine(outDir, "ciphertext.bin"); - Assert.That(File.Exists(cipherPath), Is.True, "Extract must produce ciphertext.bin."); - var expectedCipher = image.AsSpan(4096).ToArray(); - Assert.That(File.ReadAllBytes(cipherPath), Is.EqualTo(expectedCipher), - "ciphertext.bin must round-trip the opaque AES-CBC payload exactly."); - } finally { - Directory.Delete(outDir, recursive: true); - } + [TestCase("aes128")] + [TestCase("aes192")] + [TestCase("aes256")] + [Category("Interop")] + public void CreateThenExtract_RoundTripsAesPassphraseProfiles(string method) { + var content = Enumerable.Range(0, 9001).Select(i => (byte)(i * 73 + 11)).ToArray(); + var descriptor = new EcryptfsFormatDescriptor(); + using var image = new MemoryStream(CreateImage(content, method: method), writable: false); + + var entries = descriptor.List(image, null); + Assert.That(entries, Has.Count.EqualTo(1)); + Assert.That(entries[0].Name, Is.EqualTo("content.bin")); + Assert.That(entries[0].OriginalSize, Is.EqualTo(content.Length)); + Assert.That(entries[0].IsEncrypted, Is.True); + + image.Position = 0; + using var reader = new EcryptfsReader(image); + Assert.That(reader.ExtractContent("correct horse"), Is.EqualTo(content)); + } + + [Test, Category("Sad")] + public void Extract_WrongPassphrase_IsRejectedBeforePayloadDecryption() { + using var image = new MemoryStream(CreateImage([1, 2, 3, 4]), writable: false); + using var reader = new EcryptfsReader(image); + Assert.Throws(() => reader.ExtractContent("wrong horse")); + } + + [Test, Category("HappyPath")] + public void WipeAndShrink_RemoveOnlyProvenDeadBytes() { + var content = Enumerable.Range(0, 5000).Select(i => (byte)i).ToArray(); + var descriptor = new EcryptfsFormatDescriptor(); + var original = CreateImage(content); + using var image = new MemoryStream(); + image.Write(original); + image.Write(Enumerable.Repeat((byte)0xA5, 257).ToArray()); + image.Position = 0; + + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(image); + Assert.That(wiped, Is.GreaterThanOrEqualTo(257)); + Assert.That(image.ToArray().AsSpan(original.Length, 257).ToArray(), Is.All.Zero); + + image.Position = 0; + using (var reader = new EcryptfsReader(image)) + Assert.That(reader.ExtractContent("correct horse"), Is.EqualTo(content)); + + image.Position = 0; + using var shrunk = new MemoryStream(); + ((IArchiveShrinkable)descriptor).Shrink(image, shrunk); + Assert.That(shrunk.Length, Is.EqualTo(original.Length)); + using var shrunkReader = new EcryptfsReader(shrunk); + Assert.That(shrunkReader.ExtractContent("correct horse"), Is.EqualTo(content)); } - [Test, Category("Stub")] - public void Stub_DoesNotAdvertiseWriteCapability() { - var d = new EcryptfsFormatDescriptor(); - var description = d.Description.ToLowerInvariant(); - Assert.That( - description.Contains("stub") || description.Contains("opaque") - || description.Contains("skeleton") || description.Contains("detection"), - Is.True, - $"eCryptfs Description must honestly flag its stub/detection-only/opaque status. Got: '{d.Description}'."); + [Test, Category("HappyPath")] + public void Purge_LeavesValidListableEmptyLowerFile() { + var descriptor = new EcryptfsFormatDescriptor(); + using var image = new MemoryStream(CreateImage(Enumerable.Repeat((byte)0x5A, 5000).ToArray())); + + ((IArchivePurgeable)descriptor).Purge(image); + + Assert.That(image.Length, Is.EqualTo(8192)); + image.Position = 0; + Assert.That(descriptor.List(image, null), Is.Empty); + image.Position = 0; + using var reader = new EcryptfsReader(image); + Assert.That(reader.DecryptedSize, Is.Zero); + Assert.That(reader.CanonicalLength, Is.EqualTo(8192)); + } + + [Test, Category("Sad")] + public void Create_RequiresPassphrase_AndSingleRegularInput() { + var descriptor = (IArchiveCreatable)new EcryptfsFormatDescriptor(); + using var output = new MemoryStream(); + Assert.Throws(() => descriptor.Create( + output, + [ArchiveInputInfo.InMemory("a", [1])], + new FormatCreateOptions())); + + Assert.Throws(() => descriptor.Create( + output, + [ArchiveInputInfo.InMemory("a", [1]), ArchiveInputInfo.InMemory("b", [2])], + new FormatCreateOptions { Password = "pw" })); } } From e4690f70d8377aa1aa05813e50a01ef2008a5c11 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:11:41 +0200 Subject: [PATCH 097/165] Exercise TUX3 volume-length overflow guard --- Compression.Tests/Tux3/Tux3Tests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Compression.Tests/Tux3/Tux3Tests.cs b/Compression.Tests/Tux3/Tux3Tests.cs index 448adba33..db0cd2506 100644 --- a/Compression.Tests/Tux3/Tux3Tests.cs +++ b/Compression.Tests/Tux3/Tux3Tests.cs @@ -207,7 +207,7 @@ public void TruncatedDeclaredVolume_IsNeverTreatedAsFreeOrShrunk() { [Test, Category("Regression")] public void OverflowingDeclaredVolume_FailsClosed() { - var image = BuildNativeImage(volBlocks: ulong.MaxValue, blockBits: 62); + var image = BuildNativeImage(volBlocks: ulong.MaxValue, blockBits: 12); var descriptor = new Tux3FormatDescriptor(); using var layoutStream = new MemoryStream(image, writable: false); From dc18433534078cc5efb3d42324a664360eb35015 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:12:09 +0200 Subject: [PATCH 098/165] Keep Intel HEX support matrix update focused --- Hawkynt.FileFormats.FileSystems/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index 69ddc46be..86107d5ef 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -88,7 +88,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [DriveSpace 3 CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace3` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | `dmsdos` mounts the genuine `GenuineDvr3Writer` output and reads byte-exact | Win95 DVR3 CVF; a Win95 guest oracle and the HiPack codec are still open | [DriveSpace 3 notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/FileSystems/FileSystem.DriveSpace3/FORMAT-NOTES.md) | | [DriveSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` driver under QEMU mounts, reads and writes; real driver CVFs read byte-exact | MSDSP6.0 CVF; stored clusters | [DriveSpace](https://en.wikipedia.org/wiki/DriveSpace) | | [exFAT](https://en.wikipedia.org/wiki/ExFAT) | `ExFat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `exfat` mount r/w + [`fsck.exfat`](https://github.com/exfatprogs/exfatprogs) | — | [exFAT specification](https://learn.microsoft.com/windows/win32/fileio/exfat-specification) | -| [FAT Filesystem Image](https://en.wikipedia.org/wiki/File_Allocation_Table) | `Fat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount r/w + [`fsck.fat`](https://github.com/dosfstools/dosfstools) + [`mtools`](https://www.gnu.org/software/mtools) + reverse [`mkfs.vfat`](https://github.com/dosfstools/dosfstools) | FAT12/16/32 with long filenames; in-place `FatModifier` | [FAT specification](https://download.microsoft.com/download/1/6/1/161ba512-40e2-4cc9-843a-923143f3456c/fatgen103.doc) | +| [FAT Filesystem Image](https://en.wikipedia.org/wiki/File_Allocation_Table) | `Fat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount r/w + [`fsck.fat`](https://github.com/dosfstools/dosfstools) + [`mtools`](https://www.gnu.org/software/mtools/) + reverse [`mkfs.vfat`](https://github.com/dosfstools/dosfstools) | FAT12/16/32 with long filenames; in-place `FatModifier` | [FAT specification](https://download.microsoft.com/download/1/6/1/161ba512-40e2-4cc9-843a-923143f3456c/fatgen103.doc) | | [FAT+ Filesystem Image (large-file extension)](https://en.wikipedia.org/wiki/File_Allocation_Table#FAT+) | `FatPlus` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount + [`fsck.fat`](https://github.com/dosfstools/dosfstools) read the written volume | FAT+ large-file extension | [FAT+ draft](https://www.fdos.org/kernel/fatplus.txt) | | [FATX (Xbox)](https://en.wikipedia.org/wiki/FATX) | `Fatx` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Xbox FATX | [FATX (xboxdevwiki)](https://xboxdevwiki.net/FATX) | | [HPFS](https://en.wikipedia.org/wiki/High_Performance_File_System) | `Hpfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | OS/2 HPFS | [HPFS layout (OS/2 Museum)](https://www.os2museum.com/wp/the-hpfs-disk-layout/) | @@ -123,7 +123,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Minix V2 FS](https://en.wikipedia.org/wiki/MINIX_file_system) | `MinixV2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.minix`](https://github.com/util-linux/util-linux) clean after in-place edit + host-kernel loop-mount reads the added file byte-exact | 30-character names | [MINIX filesystem](https://en.wikipedia.org/wiki/MINIX_file_system) | | [NILFS v1](https://en.wikipedia.org/wiki/NILFS) | `Nilfs1` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | NILFS v1 | [NILFS](https://nilfs.sourceforge.io/) | | [NILFS2](https://en.wikipedia.org/wiki/NILFS) | `Nilfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel loop-mount reads the in-place-added file byte-exact; [`mkfs.nilfs2`](https://nilfs.sourceforge.io/) superblock re-validated | — | [NILFS](https://nilfs.sourceforge.io/) | -| [OCFS2 (Oracle Cluster Filesystem 2)](https://en.wikipedia.org/wiki/OCFS2) | `Ocfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.ocfs2 -fn`](https://github.com/markfasheh/ocfs2-tools) + `debugfs.ocfs2`; `mkfs.ocfs2` image read by the package | Edits rebuild the volume | [OCFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | +| [OCFS2 (Oracle Cluster Filesystem 2)](https://en.wikipedia.org/wiki/OCFS2) | `Ocfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.ocfs2 -fn`](https://github.com/markfasheh/ocfs2-tools) + `debugfs.ocfs2`; `mkfs.ocfs2` image read by the package | Edits rebuild the volume | [OCFS2](https://www.kernel.org/doc/html/latest/filesystems/ocfs2.html) | | [QNX4 FS](https://en.wikipedia.org/wiki/QNX4FS) | `Qnx4` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx4](https://www.kernel.org/doc/html/latest/filesystems/qnx4.html) | | [QNX6 Neutrino FS](https://en.wikipedia.org/wiki/QNX6FS) | `Qnx6` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx6](https://www.kernel.org/doc/html/latest/filesystems/qnx6.html) | | [Reiser4](https://en.wikipedia.org/wiki/Reiser4) | `Reiser4` | WORM | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; `reiser4progs` not installable here | Empty tree from `mkfs.reiser4` captures; files live in a payload area the reiser4 driver does not see, so the writer stays WORM | [Reiser4](https://reiser4.wiki.kernel.org/) | @@ -134,7 +134,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [UNIX System V FS](https://en.wikipedia.org/wiki/Unix_File_System) | `SysV` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `sysv` mount reads byte-exact | System V filesystem | [sysv](https://www.kernel.org/doc/html/latest/filesystems/sysv-fs.html) | | [VxFS (Veritas)](https://en.wikipedia.org/wiki/Veritas_File_System) | `VxFs` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | Linux `freevxfs` driver mounts the written volume | One fileset, direct extents, flat root; edits rebuild the volume | [freevxfs](https://github.com/torvalds/linux/tree/master/fs/freevxfs) | | [Xenix FS](https://en.wikipedia.org/wiki/Xenix) | `Xenix` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `sysv` (`detect_xenix`) mount reads byte-exact | — | [sysv](https://www.kernel.org/doc/html/latest/filesystems/sysv-fs.html) | -| [XFS](https://en.wikipedia.org/wiki/XFS) | `Xfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `xfs` mount r/w + [`xfs_repair -n`](https://git.kernel.org/pub/scm/fs/xfsprogs-dev.git) | XFS v5 | [XFS documentation](https://kernel.org/doc/html/latest/filesystems/xfs/index.html) | +| [XFS](https://en.wikipedia.org/wiki/XFS) | `Xfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `xfs` mount r/w + [`xfs_repair -n`](https://git.kernel.org/pub/scm/fs/xfs/xfsprogs-dev.git) | XFS v5 | [XFS documentation](https://kernel.org/doc/html/latest/filesystems/xfs/index.html) | | [ZFS](https://en.wikipedia.org/wiki/ZFS) | `Zfs` | R/W | ✅ | ✅ moving | — | ✅ | ✅ | ✅ | [`zdb -l`](https://github.com/openzfs/zfs) label path; own reader + struct-parity tests | OpenZFS v28 single-vdev profile; edits rebuild the pool | [OpenZFS documentation](https://openzfs.github.io/openzfs-docs/) | ### Apple From cc98ad2564770c262995944e21b0ed340a7349e7 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:13:21 +0200 Subject: [PATCH 099/165] - remove unrelated support-matrix drift --- Hawkynt.FileFormats.FileSystems/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index 1c11dd4cc..e29252db1 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -74,7 +74,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [TAP](https://en.wikipedia.org/wiki/ZX_Spectrum) | `Tap` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | Sinclair / Commodore tape image | [TAP (World of Spectrum)](https://worldofspectrum.net/faq/reference/formats.htm) | | [TI-TXT (MSP430)](https://en.wikipedia.org/wiki/Texas_Instruments) | `TiTxt` | R/W | ✅ | ✅ rebuild | — | — | — | ✅ | own reader + struct-parity tests | TI-TXT sparse firmware dump; edits and canonical rebuild preserve omitted address holes | [TI-TXT](https://www.ti.com/lit/pdf/slau131) | | [U-Boot uImage](https://en.wikipedia.org/wiki/Das_U-Boot) | `UImage` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | U-Boot legacy image header + payload; the writer takes one payload and the header fields from `metadata.ini`, and computes both CRCs | [U-Boot image format](https://docs.u-boot.org/en/latest/usage/fit/index.html) | -| [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | `UefiFv` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Firmware volume / FFS | [UEFI specification](https://uefi.org/specifications/) | +| [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | `UefiFv` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Firmware volume / FFS | [UEFI specification](https://uefi.org/specifications) | | [VDI](https://en.wikipedia.org/wiki/VirtualBox#Virtual_disk_image) | `Vdi` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [VirtualBox storage](https://www.virtualbox.org/manual/ch05.html) | | [VHD](https://en.wikipedia.org/wiki/VHD_(file_format)) | `Vhd` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Standalone fixed and dynamic; differencing requires a parent chain and is rejected; edits go to the inner filesystem | [Microsoft VHD](https://learn.microsoft.com/windows-server/virtualization/hyper-v/manage/manage-hyper-v-virtual-hard-disks) | | [VHDX](https://en.wikipedia.org/wiki/VHD_(file_format)#VHDX) | `Vhdx` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [MS-VHDX](https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-vhdx/) | @@ -86,9 +86,9 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | --- | --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | --- | --- | --- | | [DoubleSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DoubleSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` driver under QEMU mounts, reads and writes; real driver CVFs read byte-exact | MSDBL6.0 CVF; stored clusters | [DoubleSpace](https://en.wikipedia.org/wiki/DriveSpace) | | [DriveSpace 3 CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace3` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | `dmsdos` mounts the genuine `GenuineDvr3Writer` output and reads byte-exact | Win95 DVR3 CVF; a Win95 guest oracle and the HiPack codec are still open | [DriveSpace 3 notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/FileSystems/FileSystem.DriveSpace3/FORMAT-NOTES.md) | -| [DriveSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` mounts the DoubleSpace / DriveSpace CVFs; the `dmsdos` driver mounts the DriveSpace 3 and Stacker volumes. | MSDSP6.0 CVF; stored clusters | [DriveSpace](https://en.wikipedia.org/wiki/DriveSpace) | +| [DriveSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` driver under QEMU mounts, reads and writes; real driver CVFs read byte-exact | MSDSP6.0 CVF; stored clusters | [DriveSpace](https://en.wikipedia.org/wiki/DriveSpace) | | [exFAT](https://en.wikipedia.org/wiki/ExFAT) | `ExFat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `exfat` mount r/w + [`fsck.exfat`](https://github.com/exfatprogs/exfatprogs) | — | [exFAT specification](https://learn.microsoft.com/windows/win32/fileio/exfat-specification) | -| [FAT Filesystem Image](https://en.wikipedia.org/wiki/File_Allocation_Table) | `Fat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount + [`fsck.fat`](https://github.com/dosfstools/dosfstools) + [`mtools`](https://www.gnu.org/software/mtools/) read the written volume | FAT12/16/32 with long filenames; in-place `FatModifier` | [FAT specification](https://download.microsoft.com/download/1/6/1/161ba512-40e2-4cc9-843a-923143f3456c/fatgen103.doc) | +| [FAT Filesystem Image](https://en.wikipedia.org/wiki/File_Allocation_Table) | `Fat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount r/w + [`fsck.fat`](https://github.com/dosfstools/dosfstools) + [`mtools`](https://www.gnu.org/software/mtools/) + reverse [`mkfs.vfat`](https://github.com/dosfstools/dosfstools) | FAT12/16/32 with long filenames; in-place `FatModifier` | [FAT specification](https://download.microsoft.com/download/1/6/1/161ba512-40e2-4cc9-843a-923143f3456c/fatgen103.doc) | | [FAT+ Filesystem Image (large-file extension)](https://en.wikipedia.org/wiki/File_Allocation_Table#FAT+) | `FatPlus` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount + [`fsck.fat`](https://github.com/dosfstools/dosfstools) read the written volume | FAT+ large-file extension | [FAT+ draft](https://www.fdos.org/kernel/fatplus.txt) | | [FATX (Xbox)](https://en.wikipedia.org/wiki/FATX) | `Fatx` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Xbox FATX | [FATX (xboxdevwiki)](https://xboxdevwiki.net/FATX) | | [HPFS](https://en.wikipedia.org/wiki/High_Performance_File_System) | `Hpfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | OS/2 HPFS | [HPFS layout (OS/2 Museum)](https://www.os2museum.com/wp/the-hpfs-disk-layout/) | @@ -114,7 +114,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [GFS (Sistina/Red Hat, original)](https://en.wikipedia.org/wiki/GFS2) | `Gfs1` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; the Linux `gfs2` driver rejects GFS1 | Sistina GFS | [GFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | | [GFS2 (Global File System 2)](https://en.wikipedia.org/wiki/GFS2) | `Gfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.gfs2 -n`](https://pagure.io/gfs2-utils) clean on populated create and after add/replace/remove; reverse `mkfs.gfs2` reader coverage | Standalone `lock_nolock`; stuffed + indirect-tree files; edits rebuild while preserving image-size floor and lock table | [GFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | | [HAMMER (DragonFly BSD)](https://en.wikipedia.org/wiki/HAMMER_(file_system)) | `Hammer` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | DragonFly kernel mount r/w under QEMU + [`hammer show` / `hammer checkmap`](https://www.dragonflybsd.org/hammer/) | Single volume, about 1 GB minimum; edits rebuild the volume | [HAMMER](https://www.dragonflybsd.org/hammer/) | -| [HAMMER2 (DragonFly BSD)](https://en.wikipedia.org/wiki/HAMMER2) | `Hammer2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | DragonFly kernel mount r/w under QEMU | — | [HAMMER2](https://www.dragonflybsd.org/hammer2/) | +| [HAMMER2 (DragonFly BSD)](https://en.wikipedia.org/wiki/HAMMER2) | `Hammer2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | DragonFly kernel mount r/w under QEMU | — | [HAMMER2](https://www.dragonflybsd.org/hammer/) | | [HTFS (SCO High Throughput File System)](https://en.wikipedia.org/wiki/High_Throughput_File_System) | `Htfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | SCO HTFS | [HTFS](https://en.wikipedia.org/wiki/High_Throughput_File_System) | | [JFS](https://en.wikipedia.org/wiki/JFS_(file_system)) | `Jfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `jfs` mount + [`fsck.jfs`](http://jfs.sourceforge.net/) | JFS2 (Linux / AIX) | [JFS project](http://jfs.sourceforge.net/) | | [JFS1 (OS/2 original IBM JFS)](https://en.wikipedia.org/wiki/JFS_(file_system)) | `Jfs1` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; the Linux `jfs` driver only knows JFS2 | OS/2 JFS1 | [JFS project](http://jfs.sourceforge.net/) | @@ -125,7 +125,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [NILFS2](https://en.wikipedia.org/wiki/NILFS) | `Nilfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel loop-mount reads the in-place-added file byte-exact; [`mkfs.nilfs2`](https://nilfs.sourceforge.io/) superblock re-validated | — | [NILFS](https://nilfs.sourceforge.io/) | | [OCFS2 (Oracle Cluster Filesystem 2)](https://en.wikipedia.org/wiki/OCFS2) | `Ocfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.ocfs2 -fn`](https://github.com/markfasheh/ocfs2-tools) + `debugfs.ocfs2`; `mkfs.ocfs2` image read by the package | Edits rebuild the volume | [OCFS2](https://www.kernel.org/doc/html/latest/filesystems/ocfs2.html) | | [QNX4 FS](https://en.wikipedia.org/wiki/QNX4FS) | `Qnx4` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx4](https://www.kernel.org/doc/html/latest/filesystems/qnx4.html) | -| [QNX6 Neutrino FS](https://en.wikipedia.org/wiki/QNX6FS) | `Qnx6` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx6](https://www.kernel.org/doc/html/latest/filesystems/qnx6.html) | +| [QNX6 Neutrino FS](https://en.wikipedia.org/wiki/QNX6FS) | `Qnx6` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx6](https://en.wikipedia.org/wiki/QNX6FS) | | [Reiser4](https://en.wikipedia.org/wiki/Reiser4) | `Reiser4` | WORM | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; `reiser4progs` not installable here | Empty tree from `mkfs.reiser4` captures; files live in a payload area the reiser4 driver does not see, so the writer stays WORM | [Reiser4](https://reiser4.wiki.kernel.org/) | | [ReiserFS](https://en.wikipedia.org/wiki/ReiserFS) | `ReiserFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`reiserfsck`](https://git.kernel.org/pub/scm/linux/kernel/git/jeffm/reiserfsprogs.git), also after mutation | ReiserFS 3.6; edits rebuild the tree | [ReiserFS documentation](https://www.kernel.org/doc/html/latest/filesystems/reiserfs.html) | | [TUX2](https://en.wikipedia.org/wiki/Tux2) | `Tux2` | R | — | — | — | — | — | — | own reader + struct-parity tests | Phase-tree filesystem | [Tux2](https://en.wikipedia.org/wiki/Tux2) | @@ -166,7 +166,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [JFFS2](https://en.wikipedia.org/wiki/JFFS2) | `Jffs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`mkfs.jffs2`](http://git.infradead.org/mtd-utils.git) image read byte-exact | Log-structured | [JFFS2](https://www.kernel.org/doc/html/latest/filesystems/jffs2.html) | | [LittleFS](https://github.com/littlefs-project/littlefs) | `LittleFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [littlefs](https://github.com/littlefs-project/littlefs/blob/master/SPEC.md) | | [ROMFS](https://en.wikipedia.org/wiki/Romfs) | `RomFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `romfs` mount + [7-Zip](https://www.7-zip.org/) read the written image | Edits rebuild the image | [romfs](https://www.kernel.org/doc/html/latest/filesystems/romfs.html) | -| [SmartFS](https://nuttx.apache.org/docs/latest/components/filesystem/smartfs.html) | `SmartFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [NuttX SmartFS](https://github.com/apache/nuttx/tree/master/fs/smartfs) | +| [SmartFS](https://nuttx.apache.org/docs/latest/components/filesystem/smartfs.html) | `SmartFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | NuttX; flat root; edits rebuild the volume | [NuttX SmartFS](https://github.com/apache/nuttx/tree/master/fs/smartfs) | | [SquashFS](https://en.wikipedia.org/wiki/SquashFS) | `SquashFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `squashfs` mount + [`unsquashfs`](https://github.com/plougher/squashfs-tools) | Edits rebuild the image | [SquashFS](https://docs.kernel.org/filesystems/squashfs.html) | | [UBIFS](https://en.wikipedia.org/wiki/UBIFS) | `Ubifs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Log-structured; edits rebuild the image | [UBIFS](https://www.kernel.org/doc/html/latest/filesystems/ubifs.html) | | [YAFFS2](https://en.wikipedia.org/wiki/YAFFS) | `Yaffs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [YAFFS](https://yaffs.net/) | @@ -197,7 +197,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [SCL (ZX Spectrum)](https://en.wikipedia.org/wiki/TR-DOS) | `ZxScl` | R/W | ✅ | ✅ moving | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | SCL, `SINCLAIR` magic + checksum | [SCL](https://sinclair.wiki.zxnet.co.uk/wiki/TR-DOS) | | [Sharp X68000 Human68k](https://en.wikipedia.org/wiki/Human68k) | `Human68k` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Sharp X68000 | [Human68k](https://en.wikipedia.org/wiki/Human68k) | | [TI-99/4A DSR](https://en.wikipedia.org/wiki/TI-99/4A) | `Ti99` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [TI-99/4A disk format](https://www.unige.ch/medecine/nouspikel/ti99/disks.htm) | -| [TR-DOS](https://en.wikipedia.org/wiki/TR-DOS) | `TrDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | ZX Spectrum TRD | [TR-DOS](https://en.wikipedia.org/wiki/TR-DOS) | +| [TR-DOS](https://en.wikipedia.org/wiki/TR-DOS) | `TrDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | ZX Spectrum TRD | [TR-DOS](https://sinclair.wiki.zxnet.co.uk/wiki/TR-DOS) | | [TRSDOS / LDOS](https://en.wikipedia.org/wiki/TRSDOS) | `Trsdos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [TRSDOS](https://en.wikipedia.org/wiki/TRSDOS) | | [VDFS](https://en.wikipedia.org/wiki/Gothic_(series)) | `Vdfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Gothic-engine VDFS | [VDFS](https://github.com/PhoenixTales/vdfs) | @@ -219,7 +219,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [CephFS / RADOS](https://en.wikipedia.org/wiki/Ceph_(software)) | `CephFs` | R | — | — | — | — | — | — | detection of the on-disk signature | RADOS objects only | [Ceph](https://docs.ceph.com/) | | [Dell EMC Isilon OneFS](https://en.wikipedia.org/wiki/OneFS_distributed_file_system) | `OneFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Isilon OneFS | [OneFS](https://www.dell.com/en-us/dt/storage/powerscale.htm) | | [eCryptfs](https://en.wikipedia.org/wiki/ECryptfs) | `Ecryptfs` | R | — | — | — | — | — | — | detection of the on-disk signature | Payload is encrypted; plaintext needs the FEK | [eCryptfs](https://www.kernel.org/doc/html/latest/filesystems/ecryptfs.html) | -| [GlusterFS](https://en.wikipedia.org/wiki/GlusterFS) | `GlusterFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Brick metadata only | [GlusterFS](https://docs.gluster.org/) | +| [GlusterFS](https://en.wikipedia.org/wiki/Gluster) | `GlusterFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Brick metadata only | [GlusterFS](https://docs.gluster.org/) | | [IBM Spectrum Scale / GPFS](https://en.wikipedia.org/wiki/GPFS) | `Gpfs` | R | — | — | — | — | — | — | detection of the on-disk signature | IBM Spectrum Scale | [GPFS](https://www.ibm.com/docs/en/storage-scale) | | [JuiceFS](https://juicefs.com/) | `JuiceFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Chunk objects only | [JuiceFS](https://juicefs.com/docs/community/introduction/) | | [Lustre](https://en.wikipedia.org/wiki/Lustre_(file_system)) | `Lustre` | R | — | — | — | — | — | — | detection of the on-disk signature | OST / MDT objects only | [Lustre](https://www.lustre.org/) | From de49dada131839add86fa4a899e3b6cdca6bdd42 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:13:32 +0200 Subject: [PATCH 100/165] Keep T64 maintenance claims format-wide and fail closed --- .../FileFormat.T64/T64FormatDescriptor.cs | 73 +++---------------- 1 file changed, 12 insertions(+), 61 deletions(-) diff --git a/FileFormats/FileFormat.T64/T64FormatDescriptor.cs b/FileFormats/FileFormat.T64/T64FormatDescriptor.cs index ae9aace2a..f68837eeb 100644 --- a/FileFormats/FileFormat.T64/T64FormatDescriptor.cs +++ b/FileFormats/FileFormat.T64/T64FormatDescriptor.cs @@ -1,6 +1,5 @@ #pragma warning disable CS1591 using System.Buffers; -using System.Buffers.Binary; using Compression.Registry; using static Compression.Registry.FormatHelpers; @@ -16,8 +15,7 @@ namespace FileFormat.T64; /// /// public sealed class T64FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, - IArchiveModifiable, IArchiveDefragmentable, IArchiveShrinkable, IArchivePurgeable, - IArchiveLayoutMap, IFilesystemBlockMover { + IArchiveModifiable, IArchiveDefragmentable, IArchivePurgeable, IArchiveLayoutMap, IFilesystemBlockMover { private const int HeaderSize = 64; private const int EntrySize = 32; @@ -147,7 +145,7 @@ public void Defragment(Stream archive) /// /// Defragments a T64 image. The canonical consolidate-at-start path moves - /// payloads in place and patches only their absolute data-offset fields. + /// normal payloads in place and patches only their absolute data-offset fields. /// Other layout modes retain the established rebuild behaviour, but preserve /// T64 version, tape name, load address and Commodore file type. /// @@ -164,6 +162,9 @@ public void Defragment(Stream archive, DefragOptions options) { archive.Position = 0; using var reader = new T64Reader(archive); + if (reader.Entries.Any(static e => e.EntryType != 1)) + throw new InvalidDataException("T64: moving defrag only supports normal directory records (entry type 1)."); + var entries = reader.Entries.OrderBy(static e => e.DataOffset).ToArray(); var imageSize = archive.Length; var cursor = (long)HeaderSize + reader.DirectoryEntryCount * EntrySize; @@ -196,40 +197,6 @@ public void Defragment(Stream archive, DefragOptions options) { ImageSize: archive.Length, BlockMap: EnumerateLayout(archive).ToList(), Status: "Defragmentation complete")); } - // ── IArchiveShrinkable ─────────────────────────────────────────────── - - /// - /// Rebuilds normal T64 records into the smallest byte-tight image while - /// preserving version, tape name, start addresses and Commodore file types. - /// Images containing non-normal record kinds are copied through unchanged. - /// - public void Shrink(Stream input, Stream output) { - ArgumentNullException.ThrowIfNull(input); - ArgumentNullException.ThrowIfNull(output); - if (!input.CanRead || !input.CanSeek) - throw new ArgumentException("T64: shrink input must be readable and seekable.", nameof(input)); - if (!output.CanWrite || !output.CanSeek) - throw new ArgumentException("T64: shrink output must be writable and seekable.", nameof(output)); - - input.Position = 0; - using var reader = new T64Reader(input); - if (reader.Entries.Any(static e => e.EntryType != 1)) { - CopyOriginal(input, output); - return; - } - - var rebuilt = BuildCompactImage(reader); - if (rebuilt.LongLength >= input.Length) { - CopyOriginal(input, output); - return; - } - - output.Position = 0; - output.SetLength(0); - output.Write(rebuilt); - output.Position = 0; - } - // ── IArchivePurgeable ──────────────────────────────────────────────── /// @@ -255,13 +222,17 @@ public void Purge(Stream archive) { // ── IArchiveLayoutMap / IWipeEmpty ────────────────────────────────── /// - /// Enumerates every byte of a T64 image: the fixed header, live and free - /// directory slots, live payloads, and proven dead gaps/tail bytes. + /// Enumerates every byte of a normal-record T64 image: the fixed header, live + /// and free directory slots, live payloads, and proven dead gaps/tail bytes. + /// Unknown record kinds fail closed by exposing no free-space map at all. /// public IEnumerable EnumerateLayout(Stream archive) { ArgumentNullException.ThrowIfNull(archive); archive.Position = 0; using var reader = new T64Reader(archive); + if (reader.Entries.Any(static e => e.EntryType != 1)) + yield break; + var liveBySlot = reader.Entries.ToDictionary(static e => e.DirectoryIndex); yield return new DefragBlockInfo(0, HeaderSize, DefragBlockKind.MetadataReserved, "T64 Header"); @@ -293,7 +264,7 @@ private static void DefragmentByRebuild(Stream archive, DefragOptions options) { archive.Position = 0; using var reader = new T64Reader(archive); if (reader.Entries.Any(static e => e.EntryType != 1)) - throw new NotSupportedException("T64 rebuild defrag only supports normal directory records; consolidate-at-start can move other record kinds in place."); + throw new InvalidDataException("T64: rebuild defrag only supports normal directory records (entry type 1)."); var tapeName = reader.TapeName; var version = reader.Version; @@ -324,26 +295,6 @@ private static void DefragmentByRebuild(Stream archive, DefragOptions options) { return reader.Entries.Select(e => (e.Name, reader.Extract(e))).ToArray(); } - private static byte[] BuildCompactImage(T64Reader reader) { - var writer = new T64Writer(); - foreach (var entry in reader.Entries) - writer.AddFile(entry.Name, entry.StartAddress, entry.FileType, reader.Extract(entry)); - return writer.Build(reader.TapeName, reader.Version); - } - - private static void CopyOriginal(Stream input, Stream output) { - if (ReferenceEquals(input, output)) { - input.Position = 0; - return; - } - - input.Position = 0; - output.Position = 0; - output.SetLength(0); - input.CopyTo(output); - output.Position = 0; - } - private static void EnsureMutable(Stream archive) { if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) throw new ArgumentException("T64: stream must be readable, writable and seekable.", nameof(archive)); From b26393e2cd5fbfc5f2ee48dba8b09c7e61426298 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 05:13:46 +0000 Subject: [PATCH 101/165] Refresh NRG support matrix --- .github/workflows/nrg-doc-fix.yml | 36 ----------------------- Hawkynt.FileFormats.FileSystems/README.md | 2 +- 2 files changed, 1 insertion(+), 37 deletions(-) delete mode 100644 .github/workflows/nrg-doc-fix.yml diff --git a/.github/workflows/nrg-doc-fix.yml b/.github/workflows/nrg-doc-fix.yml deleted file mode 100644 index 867515b8f..000000000 --- a/.github/workflows/nrg-doc-fix.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Refresh NRG support matrix - -on: - push: - branches: [fix/nrg-rw-maintenance] - -permissions: - contents: write - -jobs: - refresh: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Refresh NRG support row and remove this one-shot workflow - shell: python - run: | - from pathlib import Path - path = Path("Hawkynt.FileFormats.FileSystems/README.md") - text = path.read_text(encoding="utf-8") - old = "| [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) |" - new = "| [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | ✅ | ✅ rebuild | — | ✅ | — | ✅ | own reader + struct-parity tests | NRG v1/v2 reader; v2 TAO writer; named edits rebuild the embedded ISO | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) |" - if text.count(old) != 1: - raise SystemExit(f"expected exactly one NRG support row, found {text.count(old)}") - path.write_text(text.replace(old, new), encoding="utf-8") - Path(".github/workflows/nrg-doc-fix.yml").unlink() - - name: Commit support matrix refresh - shell: bash - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "Refresh NRG support matrix" - git push origin HEAD:"$GITHUB_REF_NAME" diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index d5725e500..25b91b2b7 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -67,7 +67,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Intel HEX](https://en.wikipedia.org/wiki/Intel_HEX) | `IntelHex` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | Intel HEX and Motorola S-record, normalised to `firmware.bin` + metadata; the writer takes one payload and the base and start addresses from `metadata.ini` | [Intel HEX](https://www.keil.com/support/docs/1584/_hlp_hexfile.htm) / [S-record](https://srecord.sourceforge.net/man/man5/srec_motorola.5.html) | | [MDF/MDS](https://en.wikipedia.org/wiki/Alcohol_120%25) | `Mdf` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Alcohol 120%](https://en.wikipedia.org/wiki/Alcohol_120%25) | | [Nintendo PartitionFS](https://en.wikipedia.org/wiki/Nintendo_Switch) | `Pfs0` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | Nintendo Switch PartitionFS | [PFS0 (switchbrew)](https://switchbrew.org/wiki/NCA#PFS0) | -| [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | +| [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | ✅ | ✅ rebuild | — | ✅ | — | ✅ | own reader + struct-parity tests | NRG v1/v2 reader; v2 TAO writer; named edits rebuild the embedded ISO | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | | [PSP CSO/ZSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | `Cso` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Compressed ISO; edits re-pack the affected blocks | [CSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | | [QCOW2](https://en.wikipedia.org/wiki/Qcow) | `Qcow2` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | [`qemu-img check`](https://www.qemu.org/docs/master/tools/qemu-img.html) / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [QCOW2 specification](https://www.qemu.org/docs/master/interop/qcow2.html) | | [T64](https://en.wikipedia.org/wiki/Commodore_64) | `T64` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 tape archive | [T64 (VICE)](https://vice-emu.sourceforge.io/vice_17.html) | From c7048dae9d3a4f8e1a26f64259ef6e0a2c61212d Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:13:52 +0200 Subject: [PATCH 102/165] Update firmware writer test remarks for Intel HEX R/W --- .../FirmwareHex/FirmwareContainerWriteTests.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Compression.Tests/FirmwareHex/FirmwareContainerWriteTests.cs b/Compression.Tests/FirmwareHex/FirmwareContainerWriteTests.cs index e0520427a..041f9c319 100644 --- a/Compression.Tests/FirmwareHex/FirmwareContainerWriteTests.cs +++ b/Compression.Tests/FirmwareHex/FirmwareContainerWriteTests.cs @@ -10,11 +10,10 @@ namespace Compression.Tests.FirmwareHex; /// round trip. /// /// -/// Intel HEX, TI-TXT and the legacy uImage are all fully specified and all three -/// were read-only, which made three tracked gaps in the support matrix that were -/// no harder to close than transcribing the specification. Each is a single -/// payload under a header or a base address, so they are WORM rather than R/W — -/// there is no second file to add to one. +/// Intel HEX, TI-TXT and the legacy uImage are all fully specified and originally +/// had read-only descriptors. Their creators close the write side by transcribing +/// those specifications. Intel HEX additionally supports rebuild-based editing; +/// TI-TXT and uImage remain WORM because they expose only a single payload view. /// [TestFixture] [Category("HappyPath"), Category("RoundTrip")] From 826774f92af2e5dea9681447d692373cfae12a89 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:14:03 +0200 Subject: [PATCH 103/165] Test fail-closed T64 maintenance surface --- Compression.Tests/T64/T64MaintenanceTests.cs | 41 ++++++++------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/Compression.Tests/T64/T64MaintenanceTests.cs b/Compression.Tests/T64/T64MaintenanceTests.cs index ecf6bc5e7..800ea02eb 100644 --- a/Compression.Tests/T64/T64MaintenanceTests.cs +++ b/Compression.Tests/T64/T64MaintenanceTests.cs @@ -112,28 +112,6 @@ public void WipeUnusedSpace_ZerosFreeDirectorySlotAndPayloadGaps() { } } - [Test, Category("RoundTrip")] - public void Shrink_TightPacksAndPreservesT64Metadata() { - var (input, payload) = BuildGappedImage(); - using (input) - using (var output = new MemoryStream()) { - var descriptor = new T64FormatDescriptor(); - ((IArchiveShrinkable)descriptor).Shrink(input, output); - - Assert.That(output.Length, Is.EqualTo(64 + 32 + payload.Length)); - using var reader = new T64Reader(output); - var entry = reader.Entries.Single(); - Assert.Multiple(() => { - Assert.That(reader.DirectoryEntryCount, Is.EqualTo(1)); - Assert.That(reader.Version, Is.EqualTo(0x0101)); - Assert.That(reader.TapeName, Is.EqualTo("GAPPED TAPE")); - Assert.That(entry.StartAddress, Is.EqualTo(0x2000)); - Assert.That(entry.FileType, Is.EqualTo(0x81)); - Assert.That(reader.Extract(entry), Is.EqualTo(payload)); - }); - } - } - [Test, Category("RoundTrip")] public void Purge_LeavesValidCanonicalEmptyImage() { var (stream, _) = BuildGappedImage(); @@ -153,15 +131,30 @@ public void Purge_LeavesValidCanonicalEmptyImage() { } } + [Test, Category("ErrorHandling")] + public void LayoutMap_UnknownRecordKind_FailsClosed() { + var (stream, _) = BuildGappedImage(); + using (stream) { + var raw = stream.ToArray(); + raw[64] = 2; + using var unknown = Writable(raw); + + var descriptor = new T64FormatDescriptor(); + Assert.That(descriptor.EnumerateLayout(unknown), Is.Empty, + "unknown T64 record kinds must never turn undecoded bytes into wipeable free space"); + } + } + [Test, Category("HappyPath")] - public void Descriptor_ExposesSupportedMaintenanceCapabilities() { + public void Descriptor_ExposesOnlyFormatWideMaintenanceCapabilities() { var descriptor = new T64FormatDescriptor(); Assert.Multiple(() => { Assert.That(descriptor, Is.InstanceOf()); - Assert.That(descriptor, Is.InstanceOf()); Assert.That(descriptor, Is.InstanceOf()); Assert.That(descriptor, Is.InstanceOf()); Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf(), + "normal T64 records can be rebuilt tightly, but other defined record kinds cannot yet be faithfully re-authored"); Assert.That(descriptor, Is.Not.InstanceOf(), "T64 has no selectable filesystem/block geometry to re-layout."); }); From 41d2826bc2403ee1064bc02eb1ccf5c985a97302 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:14:04 +0200 Subject: [PATCH 104/165] * document Lustre maintenance support --- Hawkynt.FileFormats.FileSystems/README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index d5725e500..2d8d81514 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -97,7 +97,6 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [ReFS](https://en.wikipedia.org/wiki/ReFS) | `Refs` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | ReFS 3.x; mutation is offline-quiescent only (see limitations). Its geometry analysis is real, but there is no creator to write a re-laid volume with, so Layout stays absent | [ReFS overview](https://learn.microsoft.com/windows-server/storage/refs/refs-overview) | | [Stacker CVF](https://en.wikipedia.org/wiki/Stac_Electronics#Stacker) | `Stacker` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | `dmsdos` mounts the genuine `GenuineStackerWriter` output and reads byte-exact | STACVOL; Genuine/Extended flavor-preserving rebuild edits | [Stacker notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/FileSystems/FileSystem.Stacker/FORMAT-NOTES.md) | | [Transactional FAT (TFAT)](https://learn.microsoft.com/previous-versions/windows/embedded/aa911939(v=msdn.10)) | `TFat` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount + [`fsck.fat`](https://github.com/dosfstools/dosfstools) read the written volume | Transaction-safe FAT (Windows CE) | [TFAT](https://learn.microsoft.com/previous-versions/windows/embedded/aa911939(v=msdn.10)) | - ### Unix / Linux | Format | Id | State | Compact | Defrag | Wipe | Shrink | Layout | Purge | Proof | Notes | Reference | @@ -172,7 +171,6 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [YAFFS2](https://en.wikipedia.org/wiki/YAFFS) | `Yaffs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [YAFFS](https://yaffs.net/) | ### Amiga, Atari, Acorn and other home computers - | Format | Id | State | Compact | Defrag | Wipe | Shrink | Layout | Purge | Proof | Notes | Reference | | --- | --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | --- | --- | --- | | [Acorn ADFS](https://en.wikipedia.org/wiki/Advanced_Disc_Filing_System) | `Adfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; the `adfs` module is absent from the QEMU guest | Acorn ADFS | [ADFS](https://en.wikipedia.org/wiki/Advanced_Disc_Filing_System) | @@ -222,7 +220,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [GlusterFS](https://en.wikipedia.org/wiki/Gluster) | `GlusterFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Brick metadata only | [GlusterFS](https://docs.gluster.org/) | | [IBM Spectrum Scale / GPFS](https://en.wikipedia.org/wiki/GPFS) | `Gpfs` | R | — | — | — | — | — | — | detection of the on-disk signature | IBM Spectrum Scale | [GPFS](https://www.ibm.com/docs/en/storage-scale) | | [JuiceFS](https://juicefs.com/) | `JuiceFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Chunk objects only | [JuiceFS](https://juicefs.com/docs/community/introduction/) | -| [Lustre](https://en.wikipedia.org/wiki/Lustre_(file_system)) | `Lustre` | R | — | — | — | — | — | — | detection of the on-disk signature | OST / MDT objects only | [Lustre](https://www.lustre.org/) | +| [Lustre](https://en.wikipedia.org/wiki/Lustre_(file_system)) | `Lustre` | R | ✅ | — | ✅ | ✅ | — | — | own reader + struct-parity tests | ldiskfs MDT/OST backing store; conservative offline free-block wipe + trailing-block shrink; namespace edits remain R/O | [Lustre](https://www.lustre.org/) | | [MooseFS](https://en.wikipedia.org/wiki/Moose_File_System) | `MooseFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Chunk-server objects only | [MooseFS](https://moosefs.com/) | | [NetApp WAFL](https://en.wikipedia.org/wiki/Write_Anywhere_File_Layout) | `Wafl` | R | — | — | — | — | — | — | detection of the on-disk signature | NetApp WAFL | [WAFL](https://en.wikipedia.org/wiki/Write_Anywhere_File_Layout) | | [NSS (Novell Storage Services)](https://en.wikipedia.org/wiki/Novell_Storage_Services) | `Nss` | WORM | ✅ | ✅ moving | ✅ | — | — | — | anchor detection derived from real OES media; see the on-disk notes | Beast / B-tree layout undocumented; the writer emits a container under its own magic, not a pool | [NSS on-disk notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/NSS-ON-DISK.md) | @@ -247,7 +245,6 @@ using var fs = new FatReader(image); foreach (var entry in fs.ListRecursive()) Console.WriteLine($"{entry.Path} {entry.Size,10} {entry.Modified:O}"); ``` - ### Open a virtual disk and inspect its inner filesystem ```csharp @@ -297,7 +294,6 @@ The **Proof** column is produced by these suites, all in `Compression.Tests`. Ex | `ExternalConformance*`, `ExternalFsInteropTests` (`Category("ExternalFsInterop")`) | `e2fsck`, `xfs_repair`, `btrfs check`, `fsck.f2fs`, `fsck.jfs`, `reiserfsck`, `fsck.minix`, `fsck.hfsplus`, hfsutils, ntfs-3g, `mkudffs`, `unsquashfs`, `mtools`, `qemu-img` accept what the package writes, and the package reads what they write. | | DOS-era drivers under QEMU / `dmsdos` | MS-DOS 6.22 `DRVSPACE` mounts the DoubleSpace / DriveSpace CVFs; the `dmsdos` driver mounts the DriveSpace 3 and Stacker volumes. | | Struct-parity unit tests | For formats with no reachable external tool, each on-disk structure the writer emits is compared field by field against the specification and read back by the package's own reader. | - Disk-image containers follow the same rule with `qemu-img`: forward `check`, raw round-trip via `convert -O raw`, and reverse — an image `qemu-img create` made is opened by the package reader. A forensic-style path builds an inner filesystem with known files, wraps it in a container, validates the container externally, then walks it back and compares bytes. ## 🧯 Filesystem-aware recovery @@ -348,4 +344,4 @@ If this project saves you time or money, consider supporting its development: ## 📜 License -Licensed under LGPL-3.0-or-later — see the repository [LICENSE](https://github.com/Hawkynt/CompressionWorkbench/blob/main/LICENSE). +Licensed under LGPL-3.0-or-later — see the repository [LICENSE](https://github.com/Hawkynt/CompressionWorkbench/blob/main/LICENSE). \ No newline at end of file From 8a8f917bdd9a3c5c2453e285a30189927a41c11b Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:15:44 +0200 Subject: [PATCH 105/165] - revert unrelated QNX6 reference drift --- Hawkynt.FileFormats.FileSystems/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index e29252db1..15e13d0c2 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -125,7 +125,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [NILFS2](https://en.wikipedia.org/wiki/NILFS) | `Nilfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel loop-mount reads the in-place-added file byte-exact; [`mkfs.nilfs2`](https://nilfs.sourceforge.io/) superblock re-validated | — | [NILFS](https://nilfs.sourceforge.io/) | | [OCFS2 (Oracle Cluster Filesystem 2)](https://en.wikipedia.org/wiki/OCFS2) | `Ocfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.ocfs2 -fn`](https://github.com/markfasheh/ocfs2-tools) + `debugfs.ocfs2`; `mkfs.ocfs2` image read by the package | Edits rebuild the volume | [OCFS2](https://www.kernel.org/doc/html/latest/filesystems/ocfs2.html) | | [QNX4 FS](https://en.wikipedia.org/wiki/QNX4FS) | `Qnx4` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx4](https://www.kernel.org/doc/html/latest/filesystems/qnx4.html) | -| [QNX6 Neutrino FS](https://en.wikipedia.org/wiki/QNX6FS) | `Qnx6` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx6](https://en.wikipedia.org/wiki/QNX6FS) | +| [QNX6 Neutrino FS](https://en.wikipedia.org/wiki/QNX6FS) | `Qnx6` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx6](https://www.kernel.org/doc/html/latest/filesystems/qnx6.html) | | [Reiser4](https://en.wikipedia.org/wiki/Reiser4) | `Reiser4` | WORM | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; `reiser4progs` not installable here | Empty tree from `mkfs.reiser4` captures; files live in a payload area the reiser4 driver does not see, so the writer stays WORM | [Reiser4](https://reiser4.wiki.kernel.org/) | | [ReiserFS](https://en.wikipedia.org/wiki/ReiserFS) | `ReiserFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`reiserfsck`](https://git.kernel.org/pub/scm/linux/kernel/git/jeffm/reiserfsprogs.git), also after mutation | ReiserFS 3.6; edits rebuild the tree | [ReiserFS documentation](https://www.kernel.org/doc/html/latest/filesystems/reiserfs.html) | | [TUX2](https://en.wikipedia.org/wiki/Tux2) | `Tux2` | R | — | — | — | — | — | — | own reader + struct-parity tests | Phase-tree filesystem | [Tux2](https://en.wikipedia.org/wiki/Tux2) | From 3ccc9bde2d2a26ec2230aa100dc68e9e4f07f3ab Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:16:15 +0200 Subject: [PATCH 106/165] chore: regenerate filesystem support matrix --- .github/workflows/regenerate-moosefs-docs.yml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/regenerate-moosefs-docs.yml diff --git a/.github/workflows/regenerate-moosefs-docs.yml b/.github/workflows/regenerate-moosefs-docs.yml new file mode 100644 index 000000000..08dcfb4ef --- /dev/null +++ b/.github/workflows/regenerate-moosefs-docs.yml @@ -0,0 +1,42 @@ +name: Regenerate MooseFS docs + +on: + push: + branches: [feature/moosefs-maintenance] + +permissions: + contents: write + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + +jobs: + regenerate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + dotnet-quality: 'preview' + - name: Rewrite filesystem support matrix + env: + CWB_WRITE_DOCS: '1' + run: >- + dotnet test Compression.Tests/Compression.Tests.csproj + --configuration Release + --filter FullyQualifiedName~FilesystemReadmeIsCurrentTests.TheSupportMatrix_SaysWhatTheCodeSays + --nologo + - name: Commit generated matrix + shell: bash + run: | + set -euo pipefail + if git diff --quiet -- Hawkynt.FileFormats.FileSystems/README.md; then + echo 'Support matrix already current.' + exit 0 + fi + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add Hawkynt.FileFormats.FileSystems/README.md + git commit -m '* regenerate filesystem support matrix' + git push From ee89d6fd7954d88d9e393ab71559268058e8990d Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:16:36 +0200 Subject: [PATCH 107/165] Cover legacy NRG v1 ETNF images --- Compression.Tests/Nrg/NrgTests.cs | 61 ++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/Compression.Tests/Nrg/NrgTests.cs b/Compression.Tests/Nrg/NrgTests.cs index a3a47fe3e..162de18ba 100644 --- a/Compression.Tests/Nrg/NrgTests.cs +++ b/Compression.Tests/Nrg/NrgTests.cs @@ -16,10 +16,14 @@ private static byte[] CreateNrg(params (string Name, byte[] Data)[] files) { return image.ToArray(); } - private static byte[] BuildOffsetTrackNrg(string fileName, byte[] data, int prefixLength = 4096) { + private static byte[] ExtractCanonicalIso(string fileName, byte[] data) { var canonical = CreateNrg((fileName, data)); - var canonicalTrailer = checked((int)BinaryPrimitives.ReadUInt64BigEndian(canonical.AsSpan(canonical.Length - 8))); - var iso = canonical.AsSpan(0, canonicalTrailer).ToArray(); + var trailerOffset = checked((int)BinaryPrimitives.ReadUInt64BigEndian(canonical.AsSpan(canonical.Length - 8))); + return canonical.AsSpan(0, trailerOffset).ToArray(); + } + + private static byte[] BuildOffsetTrackNrg(string fileName, byte[] data, int prefixLength = 4096) { + var iso = ExtractCanonicalIso(fileName, data); using var output = new MemoryStream(); output.Write(new byte[prefixLength]); @@ -32,6 +36,39 @@ private static byte[] BuildOffsetTrackNrg(string fileName, byte[] data, int pref etn2[19] = 0x00; WriteChunk(output, "ETN2"u8, etn2); + WriteCommonTrailerChunks(output); + + Span footer = stackalloc byte[12]; + "NER5"u8.CopyTo(footer); + BinaryPrimitives.WriteUInt64BigEndian(footer[4..], trailerOffset); + output.Write(footer); + return output.ToArray(); + } + + private static byte[] BuildLegacyOffsetTrackNrg(string fileName, byte[] data, int prefixLength = 4096) { + var iso = ExtractCanonicalIso(fileName, data); + + using var output = new MemoryStream(); + output.Write(new byte[prefixLength]); + output.Write(iso); + var trailerOffset = checked((uint)output.Position); + + Span etnf = stackalloc byte[20]; + BinaryPrimitives.WriteUInt32BigEndian(etnf, checked((uint)prefixLength)); + BinaryPrimitives.WriteUInt32BigEndian(etnf[4..], checked((uint)iso.Length)); + etnf[11] = 0x00; + WriteChunk(output, "ETNF"u8, etnf); + + WriteCommonTrailerChunks(output); + + Span footer = stackalloc byte[8]; + "NERO"u8.CopyTo(footer); + BinaryPrimitives.WriteUInt32BigEndian(footer[4..], trailerOffset); + output.Write(footer); + return output.ToArray(); + } + + private static void WriteCommonTrailerChunks(Stream output) { Span sinf = stackalloc byte[4]; BinaryPrimitives.WriteUInt32BigEndian(sinf, 1); WriteChunk(output, "SINF"u8, sinf); @@ -40,12 +77,6 @@ private static byte[] BuildOffsetTrackNrg(string fileName, byte[] data, int pref BinaryPrimitives.WriteUInt32BigEndian(mtyp, 0x00000400); WriteChunk(output, "MTYP"u8, mtyp); WriteChunk(output, "END!"u8, ReadOnlySpan.Empty); - - Span footer = stackalloc byte[12]; - "NER5"u8.CopyTo(footer); - BinaryPrimitives.WriteUInt64BigEndian(footer[4..], trailerOffset); - output.Write(footer); - return output.ToArray(); } private static void WriteChunk(Stream output, ReadOnlySpan id, ReadOnlySpan payload) { @@ -110,6 +141,18 @@ public void Read_UsesEtn2TrackOffset_InsteadOfAssumingOffsetZero() { Assert.That(ReadFile(image, "OFFSET.BIN"), Is.EqualTo(payload)); } + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Read_LegacyNeroEtnf_Uses32BitTrackOffset() { + var payload = "legacy-track-offset"u8.ToArray(); + var image = BuildLegacyOffsetTrackNrg("LEGACY.BIN", payload); + + using var stream = new MemoryStream(image, writable: false); + using var reader = new NrgReader(stream); + Assert.That(reader.Version, Is.EqualTo(1)); + var entry = reader.Entries.Single(e => !e.IsDirectory && e.Name.Equals("LEGACY.BIN", StringComparison.OrdinalIgnoreCase)); + Assert.That(reader.Extract(entry), Is.EqualTo(payload)); + } + [Test, Category("Regression")] public void Read_TruncatedChunkLength_FailsClosedAndFallsBackToLegacyTrackProbe() { var canonical = CreateNrg(("SAFE.BIN", "safe"u8.ToArray())); From b4f705d8b8fa5e1a816fd4cddd3294761ace800e Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:16:41 +0200 Subject: [PATCH 108/165] Validate Intel HEX EOF address field --- FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs b/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs index 51aabb216..3ad0ecf11 100644 --- a/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs +++ b/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs @@ -104,7 +104,7 @@ private static void ValidateRecordShape(byte type, ushort address, int payloadLe throw new InvalidDataException( $"IntelHex: record type 0x{type:X2} requires {expectedPayloadLength} data bytes, got {payloadLength} in '{line}'."); - if ((type is 0x02 or 0x03 or 0x04 or 0x05) && address != 0) + if ((type is 0x01 or 0x02 or 0x03 or 0x04 or 0x05) && address != 0) throw new InvalidDataException( $"IntelHex: record type 0x{type:X2} requires address 0000, got {address:X4} in '{line}'."); } From 395496d78a4dc12382ea7cfd6a88f7ca023b06d5 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:16:55 +0200 Subject: [PATCH 109/165] Make empty NRG chunk payload explicit --- FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs b/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs index d624eee6e..e45405df4 100644 --- a/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs +++ b/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs @@ -118,7 +118,7 @@ public void Create(Stream output, IReadOnlyList inputs, Format BinaryPrimitives.WriteUInt32BigEndian(mtyp, CdRomMediumType); WriteChunk(output, "MTYP"u8, mtyp); - WriteChunk(output, "END!"u8, []); + WriteChunk(output, "END!"u8, ReadOnlySpan.Empty); Span footer = stackalloc byte[12]; "NER5"u8.CopyTo(footer); From 467fff5647e484c7a1bdfee522b5412a2aca4807 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:16:57 +0200 Subject: [PATCH 110/165] Document NWFS386 R/W maintenance support --- Hawkynt.FileFormats.FileSystems/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index d5725e500..07903e7b4 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -36,7 +36,7 @@ The package bundles every `FileSystem.*` assembly and the disk-image `FileFormat | --- | --- | | **R** | Open, list and extract only. For network, distributed and encrypted formats this is detection of the on-disk signature plus whatever metadata the object carries. | | **WORM** | Read plus create a fresh image; no supported edit of an existing image. | -| **R/W** | Read plus add / replace / remove / purge on an existing image. The edit may update blocks in place or lay the volume out again — the **Notes** column says when it is the latter. | +| **R/W** | Read plus add / replace / remove / purge on an existing image. The edit may update blocks in place or lay the volume out again — the **Notes** column says which. | The six verb columns are the maintenance operations the CLI and the UI gate on, defined once in [`docs/ARCHIVE-MODEL.md`](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/ARCHIVE-MODEL.md): @@ -73,7 +73,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [T64](https://en.wikipedia.org/wiki/Commodore_64) | `T64` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 tape archive | [T64 (VICE)](https://vice-emu.sourceforge.io/vice_17.html) | | [TAP](https://en.wikipedia.org/wiki/ZX_Spectrum) | `Tap` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | Sinclair / Commodore tape image | [TAP (World of Spectrum)](https://worldofspectrum.net/faq/reference/formats.htm) | | [TI-TXT (MSP430)](https://en.wikipedia.org/wiki/Texas_Instruments) | `TiTxt` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | TI-TXT firmware dump; the writer takes one payload and the `@address` from `metadata.ini` | [TI-TXT](https://www.ti.com/lit/pdf/slau131) | -| [U-Boot uImage](https://en.wikipedia.org/wiki/Das_U-Boot) | `UImage` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | U-Boot legacy image header + payload; the writer takes one payload and the header fields from `metadata.ini`, and computes both CRCs | [U-Boot image format](https://docs.u-boot.org/en/latest/usage/fit/index.html) | +| [U-Boot uImage](https://en.wikipedia.org/wiki/Das_U-Boot) | `UImage` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | U-Boot legacy image header + payload; the writer takes one payload and the `@address` from `metadata.ini`, and computes both CRCs | [U-Boot image format](https://docs.u-boot.org/en/latest/usage/fit/index.html) | | [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | `UefiFv` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Firmware volume / FFS | [UEFI specification](https://uefi.org/specifications) | | [VDI](https://en.wikipedia.org/wiki/VirtualBox#Virtual_disk_image) | `Vdi` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [VirtualBox storage](https://www.virtualbox.org/manual/ch05.html) | | [VHD](https://en.wikipedia.org/wiki/VHD_(file_format)) | `Vhd` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Standalone fixed and dynamic; differencing requires a parent chain and is rejected; edits go to the inner filesystem | [Microsoft VHD](https://learn.microsoft.com/windows-server/virtualization/hyper-v/manage/manage-hyper-v-virtual-hard-disks) | @@ -145,7 +145,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Apple DOS 3.3](https://en.wikipedia.org/wiki/Apple_DOS) | `AppleDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | DOS 3.3, catalog at T17 S15 | [Apple DOS](https://en.wikipedia.org/wiki/Apple_DOS) | | [Apple IIgs GS/OS (2IMG)](https://en.wikipedia.org/wiki/Apple_IIGS) | `GsOs` | R/W | ✅ | ✅ rebuild | — | ✅ | — | ✅ | own reader + struct-parity tests | 2IMG wrapper delegating to the inner ProDOS / HFS / DOS 3.3 reader | [2IMG](https://apple2.org.za/gswv/a2zine/Docs/DiskImage_2MG_Info.txt) | | [Apple UCSD Pascal](https://en.wikipedia.org/wiki/UCSD_Pascal) | `ApplePascal` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | UCSD Pascal volume | [UCSD p-System](https://en.wikipedia.org/wiki/UCSD_Pascal) | -| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://www.mars.org/home/rob/proj/hfs/) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/documentation/mac/Files/Files-2.html) | +| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://www.mars.org/home/rob/proj/hfs/) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | | [HFS+](https://en.wikipedia.org/wiki/HFS_Plus) | `HfsPlus` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.hfsplus -fn`](https://packages.debian.org/sid/hfsprogs) + [hfsutils](https://www.mars.org/home/rob/proj/hfs/) + reverse `mkfs.hfsplus` | Edits rebuild the catalog | [TN1150](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | | [MFS (Macintosh File System)](https://en.wikipedia.org/wiki/Macintosh_File_System) | `Mfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Macintosh File System (1984) | [MFS](https://en.wikipedia.org/wiki/Macintosh_File_System) | | [ProDOS](https://en.wikipedia.org/wiki/Apple_ProDOS) | `ProDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [ProDOS technical reference](https://prodos8.com/docs/techref/) | @@ -227,9 +227,9 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [NetApp WAFL](https://en.wikipedia.org/wiki/Write_Anywhere_File_Layout) | `Wafl` | R | — | — | — | — | — | — | detection of the on-disk signature | NetApp WAFL | [WAFL](https://en.wikipedia.org/wiki/Write_Anywhere_File_Layout) | | [NSS (Novell Storage Services)](https://en.wikipedia.org/wiki/Novell_Storage_Services) | `Nss` | WORM | ✅ | ✅ moving | ✅ | — | — | — | anchor detection derived from real OES media; see the on-disk notes | Beast / B-tree layout undocumented; the writer emits a container under its own magic, not a pool | [NSS on-disk notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/NSS-ON-DISK.md) | | [NWFS (Novell NetWare 386 Traditional Filesystem)](https://en.wikipedia.org/wiki/NetWare_File_System) | `Nwfs` | R | — | — | — | — | — | — | detection of the on-disk signature | NetWare 286 / 3.x | [NWFS](https://en.wikipedia.org/wiki/NetWare_File_System) | -| [NWFS386 (Novell NetWare 386 raw)](https://en.wikipedia.org/wiki/NetWare_File_System) | `Nwfs386` | R | — | — | — | — | — | — | detection of the on-disk signature | NetWare 386 raw-partition structures are proprietary | [NWFS](https://en.wikipedia.org/wiki/NetWare_File_System) | +| [NWFS386 (Novell NetWare 386 Traditional Filesystem)](https://en.wikipedia.org/wiki/NetWare_File_System) | `Nwfs386` | R/W | ✅ | ✅ rebuild | — | ✅ | ✅ | ✅ | own reader + struct-parity tests | Plain single-segment DOS namespace; rebuild edits preserve block size and volume name; compression/suballocation/alternate namespaces are read-only | [Traditional NetWare partitions](https://www.novell.com/developer/documentation/vfs/vfs__enu/data/afpro44.html) | | [OrangeFS / PVFS2 DBPF](https://en.wikipedia.org/wiki/OrangeFS) | `OrangeFs` | R/W | ✅ | ✅ rebuild | — | — | — | ✅ | own reader + struct-parity tests | One DBPF storage object, not a namespace; the opaque payload is what is edited | [OrangeFS](https://github.com/waltligon/orangefs) | -| [SGI CXFS (Cluster XFS)](https://en.wikipedia.org/wiki/CXFS) | `Cxfs` | R | — | — | — | — | — | — | detection of the on-disk signature | SGI cluster XFS | [CXFS](https://en.wikipedia.org/wiki/CXFS) | +| [SGI CXFS (Cluster XFS)](https://en.wikipedia.org/wiki/CXFS) | `Cxfs` | R | — | — | — | — | — | — | detection of the on-disk signature | SGI cluster XFS | [CXFS](https://en.wikipedia.org/wiki/Cluster_File_System) | | [Tahoe-LAFS share](https://en.wikipedia.org/wiki/Tahoe-LAFS) | `TahoeLafs` | R | — | — | — | — | — | — | detection of the on-disk signature | Shares are capability-encrypted; no read-cap, no plaintext | [Tahoe-LAFS](https://tahoe-lafs.org/) | | [TFS (BBN Trans-FS)](https://en.wikipedia.org/wiki/BBN_Technologies) | `Tfs` | R | — | — | — | — | — | — | detection of the on-disk signature | BBN Trans-FS; no public on-disk specification | [BBN](https://en.wikipedia.org/wiki/BBN_Technologies) | From d79da43fce168f0b664e2eecc9608e2c409b8447 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:17:03 +0200 Subject: [PATCH 111/165] Cover malformed Intel HEX EOF address --- Compression.Tests/FirmwareHex/IntelHexRwTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Compression.Tests/FirmwareHex/IntelHexRwTests.cs b/Compression.Tests/FirmwareHex/IntelHexRwTests.cs index 70fb61593..251df01d1 100644 --- a/Compression.Tests/FirmwareHex/IntelHexRwTests.cs +++ b/Compression.Tests/FirmwareHex/IntelHexRwTests.cs @@ -28,6 +28,7 @@ public void Reader_DecodesType03AsCsShift4PlusIp_AndKeepsRegisters() { [TestCase(":0100000412E9\n:00000001FF\n", TestName = "Reader_RejectsWrongExtendedLinearPayloadLength")] [TestCase(":020001040001F8\n:00000001FF\n", TestName = "Reader_RejectsNonZeroExtendedLinearAddressField")] + [TestCase(":00000101FE\n", TestName = "Reader_RejectsNonZeroEofAddressField")] [TestCase(":00000001FF\n:00000001FF\n", TestName = "Reader_RejectsRecordAfterEof")] [Category("EdgeCase")] public void Reader_RejectsMalformedControlRecords(string text) From 6771b4326c1d30e98632386e5e1653ce720a74b6 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:17:43 +0200 Subject: [PATCH 112/165] * update UEFI FV support matrix --- Hawkynt.FileFormats.FileSystems/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index d5725e500..a4574780f 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -36,7 +36,7 @@ The package bundles every `FileSystem.*` assembly and the disk-image `FileFormat | --- | --- | | **R** | Open, list and extract only. For network, distributed and encrypted formats this is detection of the on-disk signature plus whatever metadata the object carries. | | **WORM** | Read plus create a fresh image; no supported edit of an existing image. | -| **R/W** | Read plus add / replace / remove / purge on an existing image. The edit may update blocks in place or lay the volume out again — the **Notes** column says when it is the latter. | +| **R/W** | Read plus add / replace / remove / purge on an existing image. The edit may update blocks in place or lay the volume out again — the **Notes** column says which. | The six verb columns are the maintenance operations the CLI and the UI gate on, defined once in [`docs/ARCHIVE-MODEL.md`](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/ARCHIVE-MODEL.md): @@ -64,7 +64,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [DMG](https://en.wikipedia.org/wiki/Apple_Disk_Image) | `Dmg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | — | [Apple disk images](https://developer.apple.com/library/archive/documentation/DeveloperTools/Conceptual/SoftwareDistribution4/Concepts/sd_disk_images.html) | | [EnCase EWF (E01)](https://en.wikipedia.org/wiki/EnCase#Expert_Witness_File_Format) | `Ewf` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | EnCase E01 segment chain | [libewf documentation](https://github.com/libyal/libewf/tree/main/documentation) | | [Flattened Device Tree Blob](https://en.wikipedia.org/wiki/Devicetree) | `Dtb` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Flattened Device Tree walked as a pseudo-archive | [Devicetree specification](https://www.devicetree.org/specifications/) | -| [Intel HEX](https://en.wikipedia.org/wiki/Intel_HEX) | `IntelHex` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | Intel HEX and Motorola S-record, normalised to `firmware.bin` + metadata; the writer takes one payload and the base and start addresses from `metadata.ini` | [Intel HEX](https://www.keil.com/support/docs/1584/_hlp_hexfile.htm) / [S-record](https://srecord.sourceforge.net/man/man5/srec_motorola.5.html) | +| [Intel HEX](https://en.wikipedia.org/wiki/Intel_HEX) | `IntelHex` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | Intel HEX and Motorola S-record, normalised to `firmware.bin` + metadata; the writer takes one payload and the `@address` from `metadata.ini` | [Intel HEX](https://www.keil.com/support/docs/1584/_hlp_hexfile.htm) / [S-record](https://srecord.sourceforge.net/man/man5/srec_motorola.5.html) | | [MDF/MDS](https://en.wikipedia.org/wiki/Alcohol_120%25) | `Mdf` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Alcohol 120%](https://en.wikipedia.org/wiki/Alcohol_120%25) | | [Nintendo PartitionFS](https://en.wikipedia.org/wiki/Nintendo_Switch) | `Pfs0` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | Nintendo Switch PartitionFS | [PFS0 (switchbrew)](https://switchbrew.org/wiki/NCA#PFS0) | | [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | @@ -74,7 +74,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [TAP](https://en.wikipedia.org/wiki/ZX_Spectrum) | `Tap` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | Sinclair / Commodore tape image | [TAP (World of Spectrum)](https://worldofspectrum.net/faq/reference/formats.htm) | | [TI-TXT (MSP430)](https://en.wikipedia.org/wiki/Texas_Instruments) | `TiTxt` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | TI-TXT firmware dump; the writer takes one payload and the `@address` from `metadata.ini` | [TI-TXT](https://www.ti.com/lit/pdf/slau131) | | [U-Boot uImage](https://en.wikipedia.org/wiki/Das_U-Boot) | `UImage` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | U-Boot legacy image header + payload; the writer takes one payload and the header fields from `metadata.ini`, and computes both CRCs | [U-Boot image format](https://docs.u-boot.org/en/latest/usage/fit/index.html) | -| [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | `UefiFv` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Firmware volume / FFS | [UEFI specification](https://uefi.org/specifications) | +| [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | `UefiFv` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | Firmware volume / FFS | [UEFI specification](https://uefi.org/specifications) | | [VDI](https://en.wikipedia.org/wiki/VirtualBox#Virtual_disk_image) | `Vdi` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [VirtualBox storage](https://www.virtualbox.org/manual/ch05.html) | | [VHD](https://en.wikipedia.org/wiki/VHD_(file_format)) | `Vhd` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Standalone fixed and dynamic; differencing requires a parent chain and is rejected; edits go to the inner filesystem | [Microsoft VHD](https://learn.microsoft.com/windows-server/virtualization/hyper-v/manage/manage-hyper-v-virtual-hard-disks) | | [VHDX](https://en.wikipedia.org/wiki/VHD_(file_format)#VHDX) | `Vhdx` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [MS-VHDX](https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-vhdx/) | @@ -123,7 +123,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Minix V2 FS](https://en.wikipedia.org/wiki/MINIX_file_system) | `MinixV2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.minix`](https://github.com/util-linux/util-linux) clean after in-place edit + host-kernel loop-mount reads the added file byte-exact | 30-character names | [MINIX filesystem](https://en.wikipedia.org/wiki/MINIX_file_system) | | [NILFS v1](https://en.wikipedia.org/wiki/NILFS) | `Nilfs1` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | NILFS v1 | [NILFS](https://nilfs.sourceforge.io/) | | [NILFS2](https://en.wikipedia.org/wiki/NILFS) | `Nilfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel loop-mount reads the in-place-added file byte-exact; [`mkfs.nilfs2`](https://nilfs.sourceforge.io/) superblock re-validated | — | [NILFS](https://nilfs.sourceforge.io/) | -| [OCFS2 (Oracle Cluster Filesystem 2)](https://en.wikipedia.org/wiki/OCFS2) | `Ocfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.ocfs2 -fn`](https://github.com/markfasheh/ocfs2-tools) + `debugfs.ocfs2`; `mkfs.ocfs2` image read by the package | Edits rebuild the volume | [OCFS2](https://www.kernel.org/doc/html/latest/filesystems/ocfs2.html) | +| [OCFS2 (Oracle Cluster Filesystem 2)](https://en.wikipedia.org/wiki/OCFS2) | `Ocfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.ocfs2 -fn`](https://github.com/markfasheh/ocfs2-tools) + `debugfs.ocfs2`; `mkfs.ocfs2` image read by the package | Standalone `lock_nolock`; stuffed + indirect-tree files; edits rebuild while preserving image-size floor and lock table | [OCFS2](https://www.kernel.org/doc/html/latest/filesystems/ocfs2.html) | | [QNX4 FS](https://en.wikipedia.org/wiki/QNX4FS) | `Qnx4` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx4](https://www.kernel.org/doc/html/latest/filesystems/qnx4.html) | | [QNX6 Neutrino FS](https://en.wikipedia.org/wiki/QNX6FS) | `Qnx6` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx6](https://www.kernel.org/doc/html/latest/filesystems/qnx6.html) | | [Reiser4](https://en.wikipedia.org/wiki/Reiser4) | `Reiser4` | WORM | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; `reiser4progs` not installable here | Empty tree from `mkfs.reiser4` captures; files live in a payload area the reiser4 driver does not see, so the writer stays WORM | [Reiser4](https://reiser4.wiki.kernel.org/) | @@ -145,8 +145,8 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Apple DOS 3.3](https://en.wikipedia.org/wiki/Apple_DOS) | `AppleDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | DOS 3.3, catalog at T17 S15 | [Apple DOS](https://en.wikipedia.org/wiki/Apple_DOS) | | [Apple IIgs GS/OS (2IMG)](https://en.wikipedia.org/wiki/Apple_IIGS) | `GsOs` | R/W | ✅ | ✅ rebuild | — | ✅ | — | ✅ | own reader + struct-parity tests | 2IMG wrapper delegating to the inner ProDOS / HFS / DOS 3.3 reader | [2IMG](https://apple2.org.za/gswv/a2zine/Docs/DiskImage_2MG_Info.txt) | | [Apple UCSD Pascal](https://en.wikipedia.org/wiki/UCSD_Pascal) | `ApplePascal` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | UCSD Pascal volume | [UCSD p-System](https://en.wikipedia.org/wiki/UCSD_Pascal) | -| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://www.mars.org/home/rob/proj/hfs/) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/documentation/mac/Files/Files-2.html) | -| [HFS+](https://en.wikipedia.org/wiki/HFS_Plus) | `HfsPlus` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.hfsplus -fn`](https://packages.debian.org/sid/hfsprogs) + [hfsutils](https://www.mars.org/home/rob/proj/hfs/) + reverse `mkfs.hfsplus` | Edits rebuild the catalog | [TN1150](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | +| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://github.com/robovm/hfsutils) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/documentation/mac/Files/Files-2.html) | +| [HFS+](https://en.wikipedia.org/wiki/HFS_Plus) | `HfsPlus` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.hfsplus -fn`](https://packages.debian.org/sid/hfsprogs) + [hfsutils](https://github.com/robovm/hfsutils) + reverse `mkfs.hfsplus` | Edits rebuild the catalog | [TN1150](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | | [MFS (Macintosh File System)](https://en.wikipedia.org/wiki/Macintosh_File_System) | `Mfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Macintosh File System (1984) | [MFS](https://en.wikipedia.org/wiki/Macintosh_File_System) | | [ProDOS](https://en.wikipedia.org/wiki/Apple_ProDOS) | `ProDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [ProDOS technical reference](https://prodos8.com/docs/techref/) | From bd8232bf66ad771ff8b41404e2d7f87ad8bb7f80 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:18:17 +0200 Subject: [PATCH 113/165] chore: regenerate MooseFS package docs --- .github/workflows/regenerate-moosefs-docs.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/regenerate-moosefs-docs.yml b/.github/workflows/regenerate-moosefs-docs.yml index 08dcfb4ef..d18555b40 100644 --- a/.github/workflows/regenerate-moosefs-docs.yml +++ b/.github/workflows/regenerate-moosefs-docs.yml @@ -4,6 +4,10 @@ on: push: branches: [feature/moosefs-maintenance] +concurrency: + group: regenerate-moosefs-docs + cancel-in-progress: true + permissions: contents: write @@ -27,16 +31,21 @@ jobs: --configuration Release --filter FullyQualifiedName~FilesystemReadmeIsCurrentTests.TheSupportMatrix_SaysWhatTheCodeSays --nologo - - name: Commit generated matrix + - name: Rewrite filesystem package API reference + uses: Hawkynt/RepositoryTemplate/package-readme@v1 + with: + mode: write + project: Hawkynt.FileFormats.FileSystems/Hawkynt.FileFormats.FileSystems.csproj + - name: Commit generated documentation shell: bash run: | set -euo pipefail - if git diff --quiet -- Hawkynt.FileFormats.FileSystems/README.md; then - echo 'Support matrix already current.' + if git diff --quiet -- Hawkynt.FileFormats.FileSystems/README.md Hawkynt.FileFormats.FileSystems/REFERENCE.md; then + echo 'Filesystem package documentation already current.' exit 0 fi git config user.name github-actions[bot] git config user.email 41898282+github-actions[bot]@users.noreply.github.com - git add Hawkynt.FileFormats.FileSystems/README.md - git commit -m '* regenerate filesystem support matrix' + git add Hawkynt.FileFormats.FileSystems/README.md Hawkynt.FileFormats.FileSystems/REFERENCE.md + git commit -m '* regenerate filesystem package documentation' git push From 9f56d49a0cd43945dd04b9a52736afd8d6d8761b Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:19:03 +0200 Subject: [PATCH 114/165] Preserve NRG low-level edit coverage --- .../Nrg/NrgInPlaceModifyTests.cs | 307 +++++++++++++++--- 1 file changed, 270 insertions(+), 37 deletions(-) diff --git a/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs b/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs index ddd3f6fd6..7ebf0ee52 100644 --- a/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs +++ b/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs @@ -7,9 +7,92 @@ namespace Compression.Tests.Nrg; [TestFixture] public class NrgInPlaceModifyTests { - private const int SectorSize = 2048; + private const int Iso9660SectorSize = 2048; + private const int RawSectorSize = 2352; + private const int Mode1DataOffset = 16; + private const int Ner5FooterSize = 12; + private const int NeroFooterSize = 8; - private static byte[] BuildNrg() { + private static byte[] BuildRawMode1Nrg(int sectorCount = 32) { + if (sectorCount <= 16) + sectorCount = 20; + var dataLength = sectorCount * RawSectorSize; + var image = new byte[dataLength + Ner5FooterSize]; + + for (var i = 0; i < dataLength; ++i) + image[i] = (byte)((i * 31 + 7) & 0xFF); + + ReadOnlySpan sync = [ + 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, + ]; + for (var sector = 0; sector < sectorCount; ++sector) { + sync.CopyTo(image.AsSpan(sector * RawSectorSize, 12)); + image[sector * RawSectorSize + 12] = 0; + image[sector * RawSectorSize + 13] = 0; + image[sector * RawSectorSize + 14] = (byte)sector; + image[sector * RawSectorSize + 15] = 0x01; + } + + var pvdAt = 16 * RawSectorSize + Mode1DataOffset; + image[pvdAt] = 1; + "CD001"u8.CopyTo(image.AsSpan(pvdAt + 1)); + image[pvdAt + 6] = 1; + + "NER5"u8.CopyTo(image.AsSpan(dataLength)); + BinaryPrimitives.WriteUInt64BigEndian(image.AsSpan(dataLength + 4), checked((ulong)dataLength)); + return image; + } + + private static byte[] BuildCookedNrg(int sectorCount = 32) { + if (sectorCount <= 16) + sectorCount = 20; + var dataLength = sectorCount * Iso9660SectorSize; + var image = new byte[dataLength + Ner5FooterSize]; + + for (var i = 0; i < dataLength; ++i) + image[i] = (byte)((i * 17 + 3) & 0xFF); + + var pvdAt = 16 * Iso9660SectorSize; + image[pvdAt] = 1; + "CD001"u8.CopyTo(image.AsSpan(pvdAt + 1)); + image[pvdAt + 6] = 1; + + "NER5"u8.CopyTo(image.AsSpan(dataLength)); + BinaryPrimitives.WriteUInt64BigEndian(image.AsSpan(dataLength + 4), checked((ulong)dataLength)); + return image; + } + + private static byte[] BuildRawMode1NrgV1(int sectorCount = 32) { + if (sectorCount <= 16) + sectorCount = 20; + var dataLength = sectorCount * RawSectorSize; + var image = new byte[dataLength + NeroFooterSize]; + + for (var i = 0; i < dataLength; ++i) + image[i] = (byte)((i * 31 + 7) & 0xFF); + + ReadOnlySpan sync = [ + 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, + ]; + for (var sector = 0; sector < sectorCount; ++sector) { + sync.CopyTo(image.AsSpan(sector * RawSectorSize, 12)); + image[sector * RawSectorSize + 14] = (byte)sector; + image[sector * RawSectorSize + 15] = 0x01; + } + + var pvdAt = 16 * RawSectorSize + Mode1DataOffset; + image[pvdAt] = 1; + "CD001"u8.CopyTo(image.AsSpan(pvdAt + 1)); + image[pvdAt + 6] = 1; + + "NERO"u8.CopyTo(image.AsSpan(dataLength)); + BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(dataLength + 4), checked((uint)dataLength)); + return image; + } + + private static byte[] BuildRealNrg() { var descriptor = new NrgFormatDescriptor(); using var image = new MemoryStream(); ((IArchiveCreatable)descriptor).Create( @@ -22,70 +105,220 @@ private static byte[] BuildNrg() { private static int TrailerOffset(byte[] image) => checked((int)BinaryPrimitives.ReadUInt64BigEndian(image.AsSpan(image.Length - 8))); + [Test, Category("RoundTrip")] + public void WriteSector_Raw_RewritesUserDataOnly_FramingPreserved() { + var image = BuildRawMode1Nrg(); + var original = (byte[])image.Clone(); + using var stream = new MemoryStream(image, writable: true); + var payload = Enumerable.Range(0, Iso9660SectorSize).Select(static i => (byte)(i + 1)).ToArray(); + + NrgInPlaceModifier.WriteSector(stream, 20, payload); + + var userDataAt = 20 * RawSectorSize + Mode1DataOffset; + Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); + Assert.That(image.AsSpan(20 * RawSectorSize, Mode1DataOffset).ToArray(), + Is.EqualTo(original.AsSpan(20 * RawSectorSize, Mode1DataOffset).ToArray())); + Assert.That(image.AsSpan(userDataAt + Iso9660SectorSize, RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray(), + Is.EqualTo(original.AsSpan(userDataAt + Iso9660SectorSize, RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray())); + Assert.That(image.AsSpan(image.Length - Ner5FooterSize).ToArray(), + Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize).ToArray())); + } + + [Test, Category("RoundTrip")] + public void WriteSector_Raw_OtherSectors_ByteIdentical() { + var image = BuildRawMode1Nrg(); + var original = (byte[])image.Clone(); + using var stream = new MemoryStream(image, writable: true); + var payload = Enumerable.Repeat((byte)0xA5, Iso9660SectorSize).ToArray(); + + NrgInPlaceModifier.WriteSector(stream, 20, payload); + + var sectorCount = (image.Length - Ner5FooterSize) / RawSectorSize; + for (var lba = 0; lba < sectorCount; ++lba) { + if (lba == 20) + continue; + var offset = lba * RawSectorSize; + Assert.That(image.AsSpan(offset, RawSectorSize).ToArray(), + Is.EqualTo(original.AsSpan(offset, RawSectorSize).ToArray()), $"LBA {lba} unexpectedly changed."); + } + } + + [Test, Category("RoundTrip")] + public void WriteSector_Cooked_RewritesUserDataAtSectorOffset() { + var image = BuildCookedNrg(); + var original = (byte[])image.Clone(); + using var stream = new MemoryStream(image, writable: true); + var payload = Enumerable.Range(0, Iso9660SectorSize).Select(static i => (byte)i).ToArray(); + + NrgInPlaceModifier.WriteSector(stream, 22, payload); + + var sectorAt = 22 * Iso9660SectorSize; + Assert.That(image.AsSpan(sectorAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); + Assert.That(image.AsSpan(0, sectorAt).ToArray(), Is.EqualTo(original.AsSpan(0, sectorAt).ToArray())); + Assert.That(image.AsSpan(sectorAt + Iso9660SectorSize).ToArray(), + Is.EqualTo(original.AsSpan(sectorAt + Iso9660SectorSize).ToArray())); + } + [Test, Category("Regression")] - public void DetectGeometry_UsesDescriptorOffset_NotFooterOffset() { - var image = BuildNrg(); + public void WriteSector_PastTrackEnd_RefusesInsteadOfCorruptingChunkOffsets() { + var image = BuildRawMode1Nrg(sectorCount: 20); + var original = (byte[])image.Clone(); + using var stream = new MemoryStream(); + stream.Write(image); + stream.Position = 0; + + Assert.That( + () => NrgInPlaceModifier.WriteSector(stream, 25, new byte[Iso9660SectorSize]), + Throws.TypeOf()); + Assert.That(stream.ToArray(), Is.EqualTo(original)); + } + + [Test, Category("RoundTrip")] + public void ZeroSector_WipesUserData_PreservesFramingAndFooter() { + var image = BuildRawMode1Nrg(); + var original = (byte[])image.Clone(); + using var stream = new MemoryStream(image, writable: true); + + Assert.That(NrgInPlaceModifier.ZeroSector(stream, 19), Is.True); + + var userDataAt = 19 * RawSectorSize + Mode1DataOffset; + Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(new byte[Iso9660SectorSize])); + Assert.That(image.AsSpan(19 * RawSectorSize, Mode1DataOffset).ToArray(), + Is.EqualTo(original.AsSpan(19 * RawSectorSize, Mode1DataOffset).ToArray())); + Assert.That(image.AsSpan(userDataAt + Iso9660SectorSize, RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray(), + Is.EqualTo(original.AsSpan(userDataAt + Iso9660SectorSize, RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray())); + Assert.That(image.AsSpan(image.Length - Ner5FooterSize).ToArray(), + Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize).ToArray())); + } + + [Test, Category("RoundTrip")] + public void ZeroSector_PastEof_ReturnsFalse() { + var image = BuildRawMode1Nrg(); using var stream = new MemoryStream(image, writable: true); + Assert.That(NrgInPlaceModifier.ZeroSector(stream, 9999), Is.False); + } + [Test, Category("RoundTrip")] + public void DetectGeometry_StripsNeroV1Footer_FromDataAreaLength() { + var image = BuildRawMode1NrgV1(); + using var stream = new MemoryStream(image, writable: true); var geometry = NrgInPlaceModifier.DetectGeometry(stream); Assert.Multiple(() => { - Assert.That(geometry.SectorSize, Is.EqualTo(SectorSize)); - Assert.That(geometry.DataOffset, Is.Zero); - Assert.That(geometry.DataAreaLength, Is.EqualTo(TrailerOffset(image))); - Assert.That(geometry.DataAreaLength, Is.LessThan(image.Length - 12), "chunk table is not sector data"); + Assert.That(geometry.SectorSize, Is.EqualTo(RawSectorSize)); + Assert.That(geometry.DataOffset, Is.EqualTo(Mode1DataOffset)); + Assert.That(geometry.DataAreaLength, Is.EqualTo(32 * RawSectorSize)); }); } [Test, Category("RoundTrip")] - public void WriteSector_ExistingCookedSector_PreservesWholeDescriptor() { - var image = BuildNrg(); - var trailer = TrailerOffset(image); - var descriptorBytes = image.AsSpan(trailer).ToArray(); + public void WriteSector_NeroV1_FooterPreserved() { + var image = BuildRawMode1NrgV1(); + var original = (byte[])image.Clone(); using var stream = new MemoryStream(image, writable: true); - var payload = Enumerable.Range(0, SectorSize).Select(static i => (byte)(i * 29)).ToArray(); + var payload = Enumerable.Repeat((byte)0x33, Iso9660SectorSize).ToArray(); NrgInPlaceModifier.WriteSector(stream, 20, payload); - Assert.That(image.AsSpan(20 * SectorSize, SectorSize).ToArray(), Is.EqualTo(payload)); - Assert.That(image.AsSpan(trailer).ToArray(), Is.EqualTo(descriptorBytes)); + var userDataAt = 20 * RawSectorSize + Mode1DataOffset; + Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); + Assert.That(image.AsSpan(image.Length - NeroFooterSize).ToArray(), + Is.EqualTo(original.AsSpan(original.Length - NeroFooterSize).ToArray())); + } + + [Test, Category("RoundTrip")] + public void MutateThenExtract_RoundTripsSectorBytes() { + var image = BuildRawMode1Nrg(); + using var stream = new MemoryStream(); + stream.Write(image); + var payload = Enumerable.Range(0, Iso9660SectorSize).Select(static i => (byte)(0xAA ^ i)).ToArray(); + + NrgInPlaceModifier.WriteSector(stream, 25, payload); + + stream.Position = 25L * RawSectorSize + Mode1DataOffset; + var extracted = new byte[Iso9660SectorSize]; + stream.ReadExactly(extracted); + Assert.That(extracted, Is.EqualTo(payload)); + } + + [Test, Category("Regression")] + public void DetectGeometry_UsesDescriptorOffset_NotFooterOffset() { + var image = BuildRealNrg(); + using var stream = new MemoryStream(image, writable: true); + var geometry = NrgInPlaceModifier.DetectGeometry(stream); + + Assert.Multiple(() => { + Assert.That(geometry.SectorSize, Is.EqualTo(Iso9660SectorSize)); + Assert.That(geometry.DataOffset, Is.Zero); + Assert.That(geometry.DataAreaLength, Is.EqualTo(TrailerOffset(image))); + Assert.That(geometry.DataAreaLength, Is.LessThan(image.Length - Ner5FooterSize), "chunk table is not sector data"); + }); } [Test, Category("RoundTrip")] - public void ZeroSector_ExistingCookedSector_PreservesWholeDescriptor() { - var image = BuildNrg(); + public void WriteSector_RealChunkedImage_PreservesWholeDescriptor() { + var image = BuildRealNrg(); var trailer = TrailerOffset(image); var descriptorBytes = image.AsSpan(trailer).ToArray(); using var stream = new MemoryStream(image, writable: true); + var payload = Enumerable.Range(0, Iso9660SectorSize).Select(static i => (byte)(i * 29)).ToArray(); - Assert.That(NrgInPlaceModifier.ZeroSector(stream, 20), Is.True); + NrgInPlaceModifier.WriteSector(stream, 20, payload); - Assert.That(image.AsSpan(20 * SectorSize, SectorSize).ToArray(), Is.EqualTo(new byte[SectorSize])); + Assert.That(image.AsSpan(20 * Iso9660SectorSize, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); Assert.That(image.AsSpan(trailer).ToArray(), Is.EqualTo(descriptorBytes)); } - [Test, Category("Regression")] - public void WriteSector_PastTrackEnd_RefusesInsteadOfCorruptingChunkOffsets() { - var image = BuildNrg(); - var trailer = TrailerOffset(image); - using var stream = new MemoryStream(); - stream.Write(image); - stream.Position = 0; + [Test, Category("Boundary")] + public void WriteSector_WrongUserDataLength_Throws() { + using var stream = new MemoryStream(BuildRawMode1Nrg(), writable: true); + Assert.Throws(() => NrgInPlaceModifier.WriteSector(stream, 20, new byte[100])); + } - var firstSectorPastTrack = trailer / SectorSize; - var payload = new byte[SectorSize]; + [Test, Category("Boundary")] + public void WriteSector_NegativeLba_Throws() { + using var stream = new MemoryStream(BuildRawMode1Nrg(), writable: true); + Assert.Throws(() => + NrgInPlaceModifier.WriteSector(stream, -1, new byte[Iso9660SectorSize])); + } - Assert.That( - () => NrgInPlaceModifier.WriteSector(stream, firstSectorPastTrack, payload), - Throws.TypeOf()); - Assert.That(stream.ToArray(), Is.EqualTo(image)); + [Test, Category("HappyPath")] + public void Descriptor_AdvertisesCanModify() { + var descriptor = new NrgFormatDescriptor(); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); } - [Test] - public void SectorEntryName_RoundTrips() { - var name = NrgInPlaceModifier.FormatSectorEntryName(123); + [Test, Category("HappyPath")] + public void Descriptor_ImplementsIArchiveModifiable() + => Assert.That(new NrgFormatDescriptor(), Is.InstanceOf()); + + [Test, Category("HappyPath")] + public void TryParseSectorEntryName_RoundTrips() { + var name = NrgInPlaceModifier.FormatSectorEntryName(12345); Assert.That(NrgInPlaceModifier.TryParseSectorEntryName(name, out var lba), Is.True); - Assert.That(lba, Is.EqualTo(123)); - Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("FILE.BIN", out _), Is.False); + Assert.That(lba, Is.EqualTo(12345)); + } + + [Test, Category("Boundary")] + public void TryParseSectorEntryName_BogusNames_Rejected() { + Assert.Multiple(() => { + Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("readme.txt", out _), Is.False); + Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("sector-abc.bin", out _), Is.False); + Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("sector-.bin", out _), Is.False); + Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("", out _), Is.False); + }); + } + + [Test, Category("Boundary")] + public void DetectGeometry_StripsNer5Footer_FromDataAreaLength() { + var image = BuildRawMode1Nrg(); + using var stream = new MemoryStream(image, writable: true); + var geometry = NrgInPlaceModifier.DetectGeometry(stream); + + Assert.Multiple(() => { + Assert.That(geometry.SectorSize, Is.EqualTo(RawSectorSize)); + Assert.That(geometry.DataOffset, Is.EqualTo(Mode1DataOffset)); + Assert.That(geometry.DataAreaLength, Is.EqualTo(32 * RawSectorSize)); + }); } } From 856952057f4c3ffcaf2de57b8ab5f47a49418f10 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:19:09 +0200 Subject: [PATCH 115/165] Preserve MooseFS reader API compatibility --- FileSystems/FileSystem.MooseFs/MooseFsReader.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/FileSystems/FileSystem.MooseFs/MooseFsReader.cs b/FileSystems/FileSystem.MooseFs/MooseFsReader.cs index f3532f772..b9f5ca915 100644 --- a/FileSystems/FileSystem.MooseFs/MooseFsReader.cs +++ b/FileSystems/FileSystem.MooseFs/MooseFsReader.cs @@ -41,6 +41,17 @@ public sealed class MooseFsReader : IDisposable { /// True once the MFSM prefix has been verified. public bool ValidHeader { get; private set; } + /// + /// Compatibility view of the first eight bytes after the signature, matching + /// the value older CompressionWorkbench builds exposed under this incorrect + /// name. MooseFS does not define this field as a file-id counter. + /// + [Obsolete("MooseFS metadata has no file-id counter at this location; use MetadataVersion, MetaId, MaxNodeId and NextSessionId.")] + public ulong? FileIdCounter + => _data.Length >= SignatureSize + 8 + ? BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(SignatureSize, 8)) + : null; + /// /// Packed file-format version (0x16 for 1.6, 0x20 for 2.0), /// or null for the special MFSM NEW bootstrap image. From c8ec29b4c1a511ffb4eaeb4cf5547e646c55a404 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:19:53 +0200 Subject: [PATCH 116/165] Keep NWFS386 matrix update focused --- Hawkynt.FileFormats.FileSystems/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index 07903e7b4..2ffc7a85a 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -36,7 +36,7 @@ The package bundles every `FileSystem.*` assembly and the disk-image `FileFormat | --- | --- | | **R** | Open, list and extract only. For network, distributed and encrypted formats this is detection of the on-disk signature plus whatever metadata the object carries. | | **WORM** | Read plus create a fresh image; no supported edit of an existing image. | -| **R/W** | Read plus add / replace / remove / purge on an existing image. The edit may update blocks in place or lay the volume out again — the **Notes** column says which. | +| **R/W** | Read plus add / replace / remove / purge on an existing image. The edit may update blocks in place or lay the volume out again — the **Notes** column says when it is the latter. | The six verb columns are the maintenance operations the CLI and the UI gate on, defined once in [`docs/ARCHIVE-MODEL.md`](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/ARCHIVE-MODEL.md): @@ -73,7 +73,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [T64](https://en.wikipedia.org/wiki/Commodore_64) | `T64` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 tape archive | [T64 (VICE)](https://vice-emu.sourceforge.io/vice_17.html) | | [TAP](https://en.wikipedia.org/wiki/ZX_Spectrum) | `Tap` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | Sinclair / Commodore tape image | [TAP (World of Spectrum)](https://worldofspectrum.net/faq/reference/formats.htm) | | [TI-TXT (MSP430)](https://en.wikipedia.org/wiki/Texas_Instruments) | `TiTxt` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | TI-TXT firmware dump; the writer takes one payload and the `@address` from `metadata.ini` | [TI-TXT](https://www.ti.com/lit/pdf/slau131) | -| [U-Boot uImage](https://en.wikipedia.org/wiki/Das_U-Boot) | `UImage` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | U-Boot legacy image header + payload; the writer takes one payload and the `@address` from `metadata.ini`, and computes both CRCs | [U-Boot image format](https://docs.u-boot.org/en/latest/usage/fit/index.html) | +| [U-Boot uImage](https://en.wikipedia.org/wiki/Das_U-Boot) | `UImage` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | U-Boot legacy image header + payload; the writer takes one payload and the header fields from `metadata.ini`, and computes both CRCs | [U-Boot image format](https://docs.u-boot.org/en/latest/usage/fit/index.html) | | [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | `UefiFv` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Firmware volume / FFS | [UEFI specification](https://uefi.org/specifications) | | [VDI](https://en.wikipedia.org/wiki/VirtualBox#Virtual_disk_image) | `Vdi` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [VirtualBox storage](https://www.virtualbox.org/manual/ch05.html) | | [VHD](https://en.wikipedia.org/wiki/VHD_(file_format)) | `Vhd` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Standalone fixed and dynamic; differencing requires a parent chain and is rejected; edits go to the inner filesystem | [Microsoft VHD](https://learn.microsoft.com/windows-server/virtualization/hyper-v/manage/manage-hyper-v-virtual-hard-disks) | @@ -145,7 +145,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Apple DOS 3.3](https://en.wikipedia.org/wiki/Apple_DOS) | `AppleDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | DOS 3.3, catalog at T17 S15 | [Apple DOS](https://en.wikipedia.org/wiki/Apple_DOS) | | [Apple IIgs GS/OS (2IMG)](https://en.wikipedia.org/wiki/Apple_IIGS) | `GsOs` | R/W | ✅ | ✅ rebuild | — | ✅ | — | ✅ | own reader + struct-parity tests | 2IMG wrapper delegating to the inner ProDOS / HFS / DOS 3.3 reader | [2IMG](https://apple2.org.za/gswv/a2zine/Docs/DiskImage_2MG_Info.txt) | | [Apple UCSD Pascal](https://en.wikipedia.org/wiki/UCSD_Pascal) | `ApplePascal` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | UCSD Pascal volume | [UCSD p-System](https://en.wikipedia.org/wiki/UCSD_Pascal) | -| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://www.mars.org/home/rob/proj/hfs/) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | +| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://www.mars.org/home/rob/proj/hfs/) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/documentation/mac/Files/Files-2.html) | | [HFS+](https://en.wikipedia.org/wiki/HFS_Plus) | `HfsPlus` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.hfsplus -fn`](https://packages.debian.org/sid/hfsprogs) + [hfsutils](https://www.mars.org/home/rob/proj/hfs/) + reverse `mkfs.hfsplus` | Edits rebuild the catalog | [TN1150](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | | [MFS (Macintosh File System)](https://en.wikipedia.org/wiki/Macintosh_File_System) | `Mfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Macintosh File System (1984) | [MFS](https://en.wikipedia.org/wiki/Macintosh_File_System) | | [ProDOS](https://en.wikipedia.org/wiki/Apple_ProDOS) | `ProDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [ProDOS technical reference](https://prodos8.com/docs/techref/) | @@ -229,7 +229,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [NWFS (Novell NetWare 386 Traditional Filesystem)](https://en.wikipedia.org/wiki/NetWare_File_System) | `Nwfs` | R | — | — | — | — | — | — | detection of the on-disk signature | NetWare 286 / 3.x | [NWFS](https://en.wikipedia.org/wiki/NetWare_File_System) | | [NWFS386 (Novell NetWare 386 Traditional Filesystem)](https://en.wikipedia.org/wiki/NetWare_File_System) | `Nwfs386` | R/W | ✅ | ✅ rebuild | — | ✅ | ✅ | ✅ | own reader + struct-parity tests | Plain single-segment DOS namespace; rebuild edits preserve block size and volume name; compression/suballocation/alternate namespaces are read-only | [Traditional NetWare partitions](https://www.novell.com/developer/documentation/vfs/vfs__enu/data/afpro44.html) | | [OrangeFS / PVFS2 DBPF](https://en.wikipedia.org/wiki/OrangeFS) | `OrangeFs` | R/W | ✅ | ✅ rebuild | — | — | — | ✅ | own reader + struct-parity tests | One DBPF storage object, not a namespace; the opaque payload is what is edited | [OrangeFS](https://github.com/waltligon/orangefs) | -| [SGI CXFS (Cluster XFS)](https://en.wikipedia.org/wiki/CXFS) | `Cxfs` | R | — | — | — | — | — | — | detection of the on-disk signature | SGI cluster XFS | [CXFS](https://en.wikipedia.org/wiki/Cluster_File_System) | +| [SGI CXFS (Cluster XFS)](https://en.wikipedia.org/wiki/CXFS) | `Cxfs` | R | — | — | — | — | — | — | detection of the on-disk signature | SGI cluster XFS | [CXFS](https://en.wikipedia.org/wiki/CXFS) | | [Tahoe-LAFS share](https://en.wikipedia.org/wiki/Tahoe-LAFS) | `TahoeLafs` | R | — | — | — | — | — | — | detection of the on-disk signature | Shares are capability-encrypted; no read-cap, no plaintext | [Tahoe-LAFS](https://tahoe-lafs.org/) | | [TFS (BBN Trans-FS)](https://en.wikipedia.org/wiki/BBN_Technologies) | `Tfs` | R | — | — | — | — | — | — | detection of the on-disk signature | BBN Trans-FS; no public on-disk specification | [BBN](https://en.wikipedia.org/wiki/BBN_Technologies) | From 1996fde6d749bb285b3b991b96d826542413f2d2 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:20:09 +0200 Subject: [PATCH 117/165] * keep UEFI FV matrix update focused --- Hawkynt.FileFormats.FileSystems/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index a4574780f..2f270d5dc 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -36,7 +36,7 @@ The package bundles every `FileSystem.*` assembly and the disk-image `FileFormat | --- | --- | | **R** | Open, list and extract only. For network, distributed and encrypted formats this is detection of the on-disk signature plus whatever metadata the object carries. | | **WORM** | Read plus create a fresh image; no supported edit of an existing image. | -| **R/W** | Read plus add / replace / remove / purge on an existing image. The edit may update blocks in place or lay the volume out again — the **Notes** column says which. | +| **R/W** | Read plus add / replace / remove / purge on an existing image. The edit may update blocks in place or lay the volume out again — the **Notes** column says when it is the latter. | The six verb columns are the maintenance operations the CLI and the UI gate on, defined once in [`docs/ARCHIVE-MODEL.md`](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/ARCHIVE-MODEL.md): @@ -64,7 +64,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [DMG](https://en.wikipedia.org/wiki/Apple_Disk_Image) | `Dmg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | — | [Apple disk images](https://developer.apple.com/library/archive/documentation/DeveloperTools/Conceptual/SoftwareDistribution4/Concepts/sd_disk_images.html) | | [EnCase EWF (E01)](https://en.wikipedia.org/wiki/EnCase#Expert_Witness_File_Format) | `Ewf` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | EnCase E01 segment chain | [libewf documentation](https://github.com/libyal/libewf/tree/main/documentation) | | [Flattened Device Tree Blob](https://en.wikipedia.org/wiki/Devicetree) | `Dtb` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Flattened Device Tree walked as a pseudo-archive | [Devicetree specification](https://www.devicetree.org/specifications/) | -| [Intel HEX](https://en.wikipedia.org/wiki/Intel_HEX) | `IntelHex` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | Intel HEX and Motorola S-record, normalised to `firmware.bin` + metadata; the writer takes one payload and the `@address` from `metadata.ini` | [Intel HEX](https://www.keil.com/support/docs/1584/_hlp_hexfile.htm) / [S-record](https://srecord.sourceforge.net/man/man5/srec_motorola.5.html) | +| [Intel HEX](https://en.wikipedia.org/wiki/Intel_HEX) | `IntelHex` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | Intel HEX and Motorola S-record, normalised to `firmware.bin` + metadata; the writer takes one payload and the base and start addresses from `metadata.ini` | [Intel HEX](https://www.keil.com/support/docs/1584/_hlp_hexfile.htm) / [S-record](https://srecord.sourceforge.net/man/man5/srec_motorola.5.html) | | [MDF/MDS](https://en.wikipedia.org/wiki/Alcohol_120%25) | `Mdf` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Alcohol 120%](https://en.wikipedia.org/wiki/Alcohol_120%25) | | [Nintendo PartitionFS](https://en.wikipedia.org/wiki/Nintendo_Switch) | `Pfs0` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | Nintendo Switch PartitionFS | [PFS0 (switchbrew)](https://switchbrew.org/wiki/NCA#PFS0) | | [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | @@ -86,7 +86,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | --- | --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | --- | --- | --- | | [DoubleSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DoubleSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` driver under QEMU mounts, reads and writes; real driver CVFs read byte-exact | MSDBL6.0 CVF; stored clusters | [DoubleSpace](https://en.wikipedia.org/wiki/DriveSpace) | | [DriveSpace 3 CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace3` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | `dmsdos` mounts the genuine `GenuineDvr3Writer` output and reads byte-exact | Win95 DVR3 CVF; a Win95 guest oracle and the HiPack codec are still open | [DriveSpace 3 notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/FileSystems/FileSystem.DriveSpace3/FORMAT-NOTES.md) | -| [DriveSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` driver under QEMU mounts, reads and writes; real driver CVFs read byte-exact | MSDSP6.0 CVF; stored clusters | [DriveSpace](https://en.wikipedia.org/wiki/DriveSpace) | +| [DriveSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` mounts the DoubleSpace / DriveSpace CVFs; real driver CVFs read byte-exact | MSDSP6.0 CVF; stored clusters | [DriveSpace](https://en.wikipedia.org/wiki/DriveSpace) | | [exFAT](https://en.wikipedia.org/wiki/ExFAT) | `ExFat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `exfat` mount r/w + [`fsck.exfat`](https://github.com/exfatprogs/exfatprogs) | — | [exFAT specification](https://learn.microsoft.com/windows/win32/fileio/exfat-specification) | | [FAT Filesystem Image](https://en.wikipedia.org/wiki/File_Allocation_Table) | `Fat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount r/w + [`fsck.fat`](https://github.com/dosfstools/dosfstools) + [`mtools`](https://www.gnu.org/software/mtools/) + reverse [`mkfs.vfat`](https://github.com/dosfstools/dosfstools) | FAT12/16/32 with long filenames; in-place `FatModifier` | [FAT specification](https://download.microsoft.com/download/1/6/1/161ba512-40e2-4cc9-843a-923143f3456c/fatgen103.doc) | | [FAT+ Filesystem Image (large-file extension)](https://en.wikipedia.org/wiki/File_Allocation_Table#FAT+) | `FatPlus` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount + [`fsck.fat`](https://github.com/dosfstools/dosfstools) read the written volume | FAT+ large-file extension | [FAT+ draft](https://www.fdos.org/kernel/fatplus.txt) | @@ -123,7 +123,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Minix V2 FS](https://en.wikipedia.org/wiki/MINIX_file_system) | `MinixV2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.minix`](https://github.com/util-linux/util-linux) clean after in-place edit + host-kernel loop-mount reads the added file byte-exact | 30-character names | [MINIX filesystem](https://en.wikipedia.org/wiki/MINIX_file_system) | | [NILFS v1](https://en.wikipedia.org/wiki/NILFS) | `Nilfs1` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | NILFS v1 | [NILFS](https://nilfs.sourceforge.io/) | | [NILFS2](https://en.wikipedia.org/wiki/NILFS) | `Nilfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel loop-mount reads the in-place-added file byte-exact; [`mkfs.nilfs2`](https://nilfs.sourceforge.io/) superblock re-validated | — | [NILFS](https://nilfs.sourceforge.io/) | -| [OCFS2 (Oracle Cluster Filesystem 2)](https://en.wikipedia.org/wiki/OCFS2) | `Ocfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.ocfs2 -fn`](https://github.com/markfasheh/ocfs2-tools) + `debugfs.ocfs2`; `mkfs.ocfs2` image read by the package | Standalone `lock_nolock`; stuffed + indirect-tree files; edits rebuild while preserving image-size floor and lock table | [OCFS2](https://www.kernel.org/doc/html/latest/filesystems/ocfs2.html) | +| [OCFS2 (Oracle Cluster Filesystem 2)](https://en.wikipedia.org/wiki/OCFS2) | `Ocfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.ocfs2 -fn`](https://github.com/markfasheh/ocfs2-tools) + `debugfs.ocfs2`; `mkfs.ocfs2` image read by the package | Edits rebuild the volume | [OCFS2](https://www.kernel.org/doc/html/latest/filesystems/ocfs2.html) | | [QNX4 FS](https://en.wikipedia.org/wiki/QNX4FS) | `Qnx4` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx4](https://www.kernel.org/doc/html/latest/filesystems/qnx4.html) | | [QNX6 Neutrino FS](https://en.wikipedia.org/wiki/QNX6FS) | `Qnx6` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [qnx6](https://www.kernel.org/doc/html/latest/filesystems/qnx6.html) | | [Reiser4](https://en.wikipedia.org/wiki/Reiser4) | `Reiser4` | WORM | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; `reiser4progs` not installable here | Empty tree from `mkfs.reiser4` captures; files live in a payload area the reiser4 driver does not see, so the writer stays WORM | [Reiser4](https://reiser4.wiki.kernel.org/) | @@ -145,8 +145,8 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Apple DOS 3.3](https://en.wikipedia.org/wiki/Apple_DOS) | `AppleDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | DOS 3.3, catalog at T17 S15 | [Apple DOS](https://en.wikipedia.org/wiki/Apple_DOS) | | [Apple IIgs GS/OS (2IMG)](https://en.wikipedia.org/wiki/Apple_IIGS) | `GsOs` | R/W | ✅ | ✅ rebuild | — | ✅ | — | ✅ | own reader + struct-parity tests | 2IMG wrapper delegating to the inner ProDOS / HFS / DOS 3.3 reader | [2IMG](https://apple2.org.za/gswv/a2zine/Docs/DiskImage_2MG_Info.txt) | | [Apple UCSD Pascal](https://en.wikipedia.org/wiki/UCSD_Pascal) | `ApplePascal` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | UCSD Pascal volume | [UCSD p-System](https://en.wikipedia.org/wiki/UCSD_Pascal) | -| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://github.com/robovm/hfsutils) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/documentation/mac/Files/Files-2.html) | -| [HFS+](https://en.wikipedia.org/wiki/HFS_Plus) | `HfsPlus` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.hfsplus -fn`](https://packages.debian.org/sid/hfsprogs) + [hfsutils](https://github.com/robovm/hfsutils) + reverse `mkfs.hfsplus` | Edits rebuild the catalog | [TN1150](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | +| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://www.mars.org/home/rob/proj/hfs/) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/documentation/mac/Files/Files-2.html) | +| [HFS+](https://en.wikipedia.org/wiki/HFS_Plus) | `HfsPlus` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.hfsplus -fn`](https://packages.debian.org/sid/hfsprogs) + [hfsutils](https://www.mars.org/home/rob/proj/hfs/) + reverse `mkfs.hfsplus` | Edits rebuild the catalog | [TN1150](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | | [MFS (Macintosh File System)](https://en.wikipedia.org/wiki/Macintosh_File_System) | `Mfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Macintosh File System (1984) | [MFS](https://en.wikipedia.org/wiki/Macintosh_File_System) | | [ProDOS](https://en.wikipedia.org/wiki/Apple_ProDOS) | `ProDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [ProDOS technical reference](https://prodos8.com/docs/techref/) | @@ -306,7 +306,7 @@ Disk-image containers follow the same rule with `qemu-img`: forward `check`, raw ```csharp using var fs = File.OpenRead("sdcard.img"); -var hits = new FilesystemCarver().CarveStream(fs); +var hits = FilesystemCarver.CarveStream(fs); foreach (var c in hits) { var result = FilesystemExtractor.ExtractCarved(fs, c, $"out/{c.FormatId}_0x{c.ByteOffset:X}"); Console.WriteLine($"{c.FormatId}: {result.FilesExtracted} files, {result.FilesFailed} failed"); From 875333ff5ea58e43f2f668e338af68bb0cfbab2c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:20:21 +0200 Subject: [PATCH 118/165] Document eCryptfs write and maintenance support --- Hawkynt.FileFormats.FileSystems/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index d5725e500..c7fe6f70c 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -130,7 +130,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [ReiserFS](https://en.wikipedia.org/wiki/ReiserFS) | `ReiserFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`reiserfsck`](https://git.kernel.org/pub/scm/linux/kernel/git/jeffm/reiserfsprogs.git), also after mutation | ReiserFS 3.6; edits rebuild the tree | [ReiserFS documentation](https://www.kernel.org/doc/html/latest/filesystems/reiserfs.html) | | [TUX2](https://en.wikipedia.org/wiki/Tux2) | `Tux2` | R | — | — | — | — | — | — | own reader + struct-parity tests | Phase-tree filesystem | [Tux2](https://en.wikipedia.org/wiki/Tux2) | | [TUX3](https://en.wikipedia.org/wiki/Tux3) | `Tux3` | R | — | — | — | — | — | — | own reader + struct-parity tests | — | [Tux3](https://github.com/OGAWAHirofumi/tux3) | -| [UFS](https://en.wikipedia.org/wiki/Unix_File_System) | `Ufs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | FreeBSD kernel mount r/w + [`fsck_ffs`](https://man.freebsd.org/cgi/man.cgi?query=fsck_ffs) under QEMU | UFS1 / FFS | [UFS](https://en.wikipedia.org/wiki/Unix_File_System) | +| [UFS](https://en.wikipedia.org/wiki/Unix_File_System) | `Ufs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | FreeBSD kernel mount r/w + [`fsck_ffs`](https://man.freebsd.org/cgi/man.cgi?query=fsck_ffs) under QEMU | UFS1 / FFS | [UFS](https://en.wikipedia.org/wiki/UFS) | | [UNIX System V FS](https://en.wikipedia.org/wiki/Unix_File_System) | `SysV` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `sysv` mount reads byte-exact | System V filesystem | [sysv](https://www.kernel.org/doc/html/latest/filesystems/sysv-fs.html) | | [VxFS (Veritas)](https://en.wikipedia.org/wiki/Veritas_File_System) | `VxFs` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | Linux `freevxfs` driver mounts the written volume | One fileset, direct extents, flat root; edits rebuild the volume | [freevxfs](https://github.com/torvalds/linux/tree/master/fs/freevxfs) | | [Xenix FS](https://en.wikipedia.org/wiki/Xenix) | `Xenix` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `sysv` (`detect_xenix`) mount reads byte-exact | — | [sysv](https://www.kernel.org/doc/html/latest/filesystems/sysv-fs.html) | @@ -145,7 +145,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Apple DOS 3.3](https://en.wikipedia.org/wiki/Apple_DOS) | `AppleDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | DOS 3.3, catalog at T17 S15 | [Apple DOS](https://en.wikipedia.org/wiki/Apple_DOS) | | [Apple IIgs GS/OS (2IMG)](https://en.wikipedia.org/wiki/Apple_IIGS) | `GsOs` | R/W | ✅ | ✅ rebuild | — | ✅ | — | ✅ | own reader + struct-parity tests | 2IMG wrapper delegating to the inner ProDOS / HFS / DOS 3.3 reader | [2IMG](https://apple2.org.za/gswv/a2zine/Docs/DiskImage_2MG_Info.txt) | | [Apple UCSD Pascal](https://en.wikipedia.org/wiki/UCSD_Pascal) | `ApplePascal` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | UCSD Pascal volume | [UCSD p-System](https://en.wikipedia.org/wiki/UCSD_Pascal) | -| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://www.mars.org/home/rob/proj/hfs/) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/documentation/mac/Files/Files-2.html) | +| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://www.mars.org/home/rob/proj/hfs/) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | | [HFS+](https://en.wikipedia.org/wiki/HFS_Plus) | `HfsPlus` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.hfsplus -fn`](https://packages.debian.org/sid/hfsprogs) + [hfsutils](https://www.mars.org/home/rob/proj/hfs/) + reverse `mkfs.hfsplus` | Edits rebuild the catalog | [TN1150](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | | [MFS (Macintosh File System)](https://en.wikipedia.org/wiki/Macintosh_File_System) | `Mfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Macintosh File System (1984) | [MFS](https://en.wikipedia.org/wiki/Macintosh_File_System) | | [ProDOS](https://en.wikipedia.org/wiki/Apple_ProDOS) | `ProDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [ProDOS technical reference](https://prodos8.com/docs/techref/) | @@ -182,7 +182,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [ATR (Atari 8-bit)](https://en.wikipedia.org/wiki/Atari_DOS) | `Atari8` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | ATR, Atari DOS 2 VTOC | [Atari DOS](https://en.wikipedia.org/wiki/Atari_DOS) | | [BBC DFS](https://en.wikipedia.org/wiki/Disc_Filing_System) | `Bbc` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | BBC DFS | [DFS](https://en.wikipedia.org/wiki/Disc_Filing_System) | | [CP/M 2.2 (8" SSSD)](https://en.wikipedia.org/wiki/CP/M) | `Cpm` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | cpmtools [`cpmls` / `cpmcp`](http://www.moria.de/~michael/cpmtools/) read the written IBM-3740 image | CP/M 2.2, 8-inch SSSD | [CP/M format notes](https://www.seasip.info/Cpm/format22.html) | -| [CPC DSK](https://en.wikipedia.org/wiki/Amstrad_CPC) | `CpcDsk` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; cpmtools / libdsk expect a different EDSK geometry | MV - CPCEMU DSK | [DSK format](https://www.cpcwiki.eu/index.php/Format:DSK_disk_image_file_format) | +| [CPC DSK](https://en.wikipedia.org/wiki/Amstrad_CPC) | `CpcDsk` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; cpmtools / libdsk expect a different EDSK geometry | MV - CPCEMU DSK | [DSK format](https://www.cpcwiki.eu/index.php?title=Format:DSK_disk_image_file_format) | | [Cromemco RDOS](https://en.wikipedia.org/wiki/Cromemco) | `Cromemco` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | RDOS | [Cromemco](https://en.wikipedia.org/wiki/Cromemco) | | [D64](https://en.wikipedia.org/wiki/Commodore_DOS) | `D64` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [VICE disk images](https://vice-emu.sourceforge.io/vice_17.html) | | [D71](https://en.wikipedia.org/wiki/Commodore_DOS) | `D71` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [VICE disk images](https://vice-emu.sourceforge.io/vice_17.html) | @@ -218,7 +218,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [BeeGFS](https://en.wikipedia.org/wiki/BeeGFS) | `BeeGfs` | R | — | — | — | — | — | — | detection of the on-disk signature | Server-side objects only; no self-contained image exists | [BeeGFS](https://www.beegfs.io/) | | [CephFS / RADOS](https://en.wikipedia.org/wiki/Ceph_(software)) | `CephFs` | R | — | — | — | — | — | — | detection of the on-disk signature | RADOS objects only | [Ceph](https://docs.ceph.com/) | | [Dell EMC Isilon OneFS](https://en.wikipedia.org/wiki/OneFS_distributed_file_system) | `OneFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Isilon OneFS | [OneFS](https://www.dell.com/en-us/dt/storage/powerscale.htm) | -| [eCryptfs](https://en.wikipedia.org/wiki/ECryptfs) | `Ecryptfs` | R | — | — | — | — | — | — | detection of the on-disk signature | Payload is encrypted; plaintext needs the FEK | [eCryptfs](https://www.kernel.org/doc/html/latest/filesystems/ecryptfs.html) | +| [eCryptfs](https://en.wikipedia.org/wiki/ECryptfs) | `Ecryptfs` | WORM | ✅ | — | ✅ | ✅ | — | ✅ | [ecryptfs-utils passphrase vector](https://github.com/dustinkirkland/ecryptfs-utils/blob/master/tests/userspace/verify-passphrase-sig.sh) + Linux kernel layout | Passphrase lower-file AES-128/192/256 read/create; private-key auth and xattr-only metadata are unsupported | [eCryptfs](https://www.kernel.org/doc/html/latest/filesystems/ecryptfs.html) | | [GlusterFS](https://en.wikipedia.org/wiki/Gluster) | `GlusterFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Brick metadata only | [GlusterFS](https://docs.gluster.org/) | | [IBM Spectrum Scale / GPFS](https://en.wikipedia.org/wiki/GPFS) | `Gpfs` | R | — | — | — | — | — | — | detection of the on-disk signature | IBM Spectrum Scale | [GPFS](https://www.ibm.com/docs/en/storage-scale) | | [JuiceFS](https://juicefs.com/) | `JuiceFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Chunk objects only | [JuiceFS](https://juicefs.com/docs/community/introduction/) | @@ -336,7 +336,7 @@ cwb recover raw.img --mode files --format Jpeg,Png - **Reiser4** remains WORM: the current writer captures the empty native tree but stores added files in a payload area the reiser4 driver does not see. It must not be promoted until the native tree writer owns those files. - **GFS2** R/W is the standalone `lock_nolock` profile with regular files in the root directory; small files are stuffed and larger files use the indirect tree. Existing-image edits rebuild the volume while preserving its size floor and lock-table value. ExHash/nested-directory writing, journal replay, cluster-lock-manager state and extended attributes remain out of scope. - **NSS** is detection only: Novell never published the format, and the writer deliberately emits a container under its own magic that no NetWare or OES release would take for a pool. What was learned from real media is in [NSS-ON-DISK.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/NSS-ON-DISK.md). -- Network, distributed and encrypted formats carry no self-contained image: a single server object, brick or share is recognised and its metadata surfaced, and nothing more is inferred. +- Network and distributed formats generally carry no self-contained image. eCryptfs is the encrypted exception handled here: a lower file is structurally self-contained, and the passphrase profile is supported for AES-128/192/256; private-key authentication and xattr-only metadata remain unsupported. - Disk-image container support and inner-filesystem support are separate capabilities; a container marked R/W edits the filesystem inside it through that filesystem's own descriptor. ## ❤️ Support From ac1e82771359233521e44fb0525a282de8ae1067 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:21:41 +0200 Subject: [PATCH 119/165] Complete NWFS386 descriptor contract --- FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs b/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs index b11bc0c51..f3dc64cb2 100644 --- a/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs +++ b/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs @@ -77,6 +77,7 @@ public sealed class Nwfs386FormatDescriptor : public IReadOnlyList MagicSignatures => []; public IReadOnlyList Methods => [new("stored", "Stored")]; + public string? TarCompressionFormatId => null; public AlgorithmFamily Family => AlgorithmFamily.Archive; public string Description => "Novell NetWare 386 Traditional filesystem. R/W rebuilds are limited to the " + From 90f71615f77db2be5c70849d23625dc70dc312f7 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 07:22:56 +0200 Subject: [PATCH 120/165] Keep eCryptfs documentation diff focused --- Hawkynt.FileFormats.FileSystems/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index c7fe6f70c..b82501b83 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -130,7 +130,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [ReiserFS](https://en.wikipedia.org/wiki/ReiserFS) | `ReiserFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`reiserfsck`](https://git.kernel.org/pub/scm/linux/kernel/git/jeffm/reiserfsprogs.git), also after mutation | ReiserFS 3.6; edits rebuild the tree | [ReiserFS documentation](https://www.kernel.org/doc/html/latest/filesystems/reiserfs.html) | | [TUX2](https://en.wikipedia.org/wiki/Tux2) | `Tux2` | R | — | — | — | — | — | — | own reader + struct-parity tests | Phase-tree filesystem | [Tux2](https://en.wikipedia.org/wiki/Tux2) | | [TUX3](https://en.wikipedia.org/wiki/Tux3) | `Tux3` | R | — | — | — | — | — | — | own reader + struct-parity tests | — | [Tux3](https://github.com/OGAWAHirofumi/tux3) | -| [UFS](https://en.wikipedia.org/wiki/Unix_File_System) | `Ufs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | FreeBSD kernel mount r/w + [`fsck_ffs`](https://man.freebsd.org/cgi/man.cgi?query=fsck_ffs) under QEMU | UFS1 / FFS | [UFS](https://en.wikipedia.org/wiki/UFS) | +| [UFS](https://en.wikipedia.org/wiki/Unix_File_System) | `Ufs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | FreeBSD kernel mount r/w + [`fsck_ffs`](https://man.freebsd.org/cgi/man.cgi?query=fsck_ffs) under QEMU | UFS1 / FFS | [UFS](https://en.wikipedia.org/wiki/Unix_File_System) | | [UNIX System V FS](https://en.wikipedia.org/wiki/Unix_File_System) | `SysV` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `sysv` mount reads byte-exact | System V filesystem | [sysv](https://www.kernel.org/doc/html/latest/filesystems/sysv-fs.html) | | [VxFS (Veritas)](https://en.wikipedia.org/wiki/Veritas_File_System) | `VxFs` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | Linux `freevxfs` driver mounts the written volume | One fileset, direct extents, flat root; edits rebuild the volume | [freevxfs](https://github.com/torvalds/linux/tree/master/fs/freevxfs) | | [Xenix FS](https://en.wikipedia.org/wiki/Xenix) | `Xenix` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `sysv` (`detect_xenix`) mount reads byte-exact | — | [sysv](https://www.kernel.org/doc/html/latest/filesystems/sysv-fs.html) | @@ -145,7 +145,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Apple DOS 3.3](https://en.wikipedia.org/wiki/Apple_DOS) | `AppleDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | DOS 3.3, catalog at T17 S15 | [Apple DOS](https://en.wikipedia.org/wiki/Apple_DOS) | | [Apple IIgs GS/OS (2IMG)](https://en.wikipedia.org/wiki/Apple_IIGS) | `GsOs` | R/W | ✅ | ✅ rebuild | — | ✅ | — | ✅ | own reader + struct-parity tests | 2IMG wrapper delegating to the inner ProDOS / HFS / DOS 3.3 reader | [2IMG](https://apple2.org.za/gswv/a2zine/Docs/DiskImage_2MG_Info.txt) | | [Apple UCSD Pascal](https://en.wikipedia.org/wiki/UCSD_Pascal) | `ApplePascal` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | UCSD Pascal volume | [UCSD p-System](https://en.wikipedia.org/wiki/UCSD_Pascal) | -| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://www.mars.org/home/rob/proj/hfs/) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | +| [HFS (Classic)](https://en.wikipedia.org/wiki/Hierarchical_File_System_(Apple)) | `Hfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`hmount` / `hls`](https://www.mars.org/home/rob/proj/hfs/) (hfsutils) list the written files | Classic HFS; edits rebuild the catalog | [Inside Macintosh: Files](https://developer.apple.com/library/archive/documentation/mac/Files/Files-2.html) | | [HFS+](https://en.wikipedia.org/wiki/HFS_Plus) | `HfsPlus` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.hfsplus -fn`](https://packages.debian.org/sid/hfsprogs) + [hfsutils](https://www.mars.org/home/rob/proj/hfs/) + reverse `mkfs.hfsplus` | Edits rebuild the catalog | [TN1150](https://developer.apple.com/library/archive/technotes/tn/tn1150.html) | | [MFS (Macintosh File System)](https://en.wikipedia.org/wiki/Macintosh_File_System) | `Mfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Macintosh File System (1984) | [MFS](https://en.wikipedia.org/wiki/Macintosh_File_System) | | [ProDOS](https://en.wikipedia.org/wiki/Apple_ProDOS) | `ProDos` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [ProDOS technical reference](https://prodos8.com/docs/techref/) | @@ -182,7 +182,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [ATR (Atari 8-bit)](https://en.wikipedia.org/wiki/Atari_DOS) | `Atari8` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | ATR, Atari DOS 2 VTOC | [Atari DOS](https://en.wikipedia.org/wiki/Atari_DOS) | | [BBC DFS](https://en.wikipedia.org/wiki/Disc_Filing_System) | `Bbc` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | BBC DFS | [DFS](https://en.wikipedia.org/wiki/Disc_Filing_System) | | [CP/M 2.2 (8" SSSD)](https://en.wikipedia.org/wiki/CP/M) | `Cpm` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | cpmtools [`cpmls` / `cpmcp`](http://www.moria.de/~michael/cpmtools/) read the written IBM-3740 image | CP/M 2.2, 8-inch SSSD | [CP/M format notes](https://www.seasip.info/Cpm/format22.html) | -| [CPC DSK](https://en.wikipedia.org/wiki/Amstrad_CPC) | `CpcDsk` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; cpmtools / libdsk expect a different EDSK geometry | MV - CPCEMU DSK | [DSK format](https://www.cpcwiki.eu/index.php?title=Format:DSK_disk_image_file_format) | +| [CPC DSK](https://en.wikipedia.org/wiki/Amstrad_CPC) | `CpcDsk` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; cpmtools / libdsk expect a different EDSK geometry | MV - CPCEMU DSK | [DSK format](https://www.cpcwiki.eu/index.php/Format:DSK_disk_image_file_format) | | [Cromemco RDOS](https://en.wikipedia.org/wiki/Cromemco) | `Cromemco` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | RDOS | [Cromemco](https://en.wikipedia.org/wiki/Cromemco) | | [D64](https://en.wikipedia.org/wiki/Commodore_DOS) | `D64` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [VICE disk images](https://vice-emu.sourceforge.io/vice_17.html) | | [D71](https://en.wikipedia.org/wiki/Commodore_DOS) | `D71` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [VICE disk images](https://vice-emu.sourceforge.io/vice_17.html) | From 6cbfef811ee0baec1449dac71fb57f3ffd92bcdf Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:10:41 +0200 Subject: [PATCH 121/165] noop --- docs/.tmp | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/.tmp diff --git a/docs/.tmp b/docs/.tmp new file mode 100644 index 000000000..e69de29bb From d9426b0b135eb39c20591686c7b3186916064bde Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:10:53 +0200 Subject: [PATCH 122/165] Remove accidental temporary file --- docs/.tmp | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/.tmp diff --git a/docs/.tmp b/docs/.tmp deleted file mode 100644 index e69de29bb..000000000 From 4ba1d45e6676ca308aea1f24baa4498c0ebb558f Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:16:31 +0200 Subject: [PATCH 123/165] Fail closed on corrupted ldiskfs allocation metadata --- .../LustreLdiskfsMaintenance.cs | 125 +++++++++++++++--- 1 file changed, 106 insertions(+), 19 deletions(-) diff --git a/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs b/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs index 27bfb5410..9c0d60d43 100644 --- a/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs +++ b/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs @@ -14,8 +14,10 @@ namespace FileSystem.Lustre; /// /// The layout facts used here come from the Linux ext4 on-disk documentation. In particular, /// block-group descriptors locate the block bitmaps, one clear bitmap bit means one free block, -/// and a BLOCK_UNINIT group does not carry a trustworthy on-disk bitmap. Lustre documents -/// ldiskfs as its ext4-derived backing filesystem. No external implementation code is copied. +/// and a BLOCK_UNINIT group does not carry a trustworthy on-disk bitmap. Destructive maintenance +/// additionally requires metadata_csum and verifies the superblock, group descriptor and block +/// bitmap checksums before trusting allocation state. Lustre documents ldiskfs as its ext4-derived +/// backing filesystem. No external implementation code is copied. /// internal static class LustreLdiskfsMaintenance { private const int SuperblockOffset = 1024; @@ -27,8 +29,10 @@ internal static class LustreLdiskfsMaintenance { private const uint IncompatJournalDev = 0x0008; private const uint IncompatMetaBg = 0x0010; private const uint Incompat64Bit = 0x0080; + private const uint IncompatCsumSeed = 0x2000; private const uint RoCompatBigalloc = 0x0200; + private const uint RoCompatMetadataCsum = 0x0400; private const uint RoCompatReadOnly = 0x1000; private const ushort BgBlockUninit = 0x0002; @@ -43,7 +47,8 @@ private readonly record struct Geometry( long DescriptorTableOffset, uint FeatureCompat, uint FeatureIncompat, - uint FeatureRoCompat); + uint FeatureRoCompat, + uint ChecksumSeed); /// /// Wipes only complete blocks that the initialized ldiskfs allocation bitmap proves free. @@ -65,20 +70,13 @@ public static long WipeFreeBlocks(Stream image) { for (uint group = 0; group < geometry.GroupCount; ++group) { ReadDescriptor(image, geometry, group, descriptor); + ValidateGroupDescriptorChecksum(geometry, group, descriptor); + var flags = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.AsSpan(0x12, 2)); if ((flags & BgBlockUninit) != 0) continue; - var bitmapBlock = ReadBlockBitmapBlock(descriptor, geometry.DescriptorSize); - if (bitmapBlock >= geometry.TotalBlocks) - throw new InvalidDataException($"Lustre/ldiskfs: group {group} block bitmap points outside the volume ({bitmapBlock})."); - - var bitmapOffset = CheckedByteOffset(bitmapBlock, geometry.BlockSize); - if (bitmapOffset + geometry.BlockSize > image.Length) - throw new InvalidDataException($"Lustre/ldiskfs: group {group} block bitmap is truncated."); - - image.Position = bitmapOffset; - image.ReadExactly(bitmap); + ReadAndValidateBlockBitmap(image, geometry, group, descriptor, bitmap); var groupFirst = (ulong)geometry.FirstDataBlock + (ulong)group * geometry.BlocksPerGroup; if (groupFirst >= geometry.TotalBlocks) @@ -187,13 +185,21 @@ private static Geometry ReadGeometry(Stream image) { var firstDataBlock = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(20, 4)); var blocksPerGroup = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(32, 4)); - if (blocksPerGroup == 0) - throw new InvalidDataException("Lustre/ldiskfs: s_blocks_per_group is zero."); + if (blocksPerGroup == 0 || (ulong)blocksPerGroup > (ulong)blockSize * 8) + throw new InvalidDataException($"Lustre/ldiskfs: invalid s_blocks_per_group value {blocksPerGroup} for {blockSize}-byte blocks."); var featureCompat = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(92, 4)); var featureIncompat = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(96, 4)); var featureRoCompat = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(100, 4)); + uint checksumSeed = 0; + if ((featureRoCompat & RoCompatMetadataCsum) != 0) { + ValidateSuperblockChecksum(superblock); + checksumSeed = (featureIncompat & IncompatCsumSeed) != 0 + ? BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(0x270, 4)) + : Crc32c(0xFFFFFFFFu, superblock.AsSpan(104, 16)); + } + ulong totalBlocks = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(4, 4)); if ((featureIncompat & Incompat64Bit) != 0) totalBlocks |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(0x150, 4)) << 32; @@ -203,10 +209,9 @@ private static Geometry ReadGeometry(Stream image) { var descriptorSize = 32; if ((featureIncompat & Incompat64Bit) != 0) { descriptorSize = BinaryPrimitives.ReadUInt16LittleEndian(superblock.AsSpan(0xFE, 2)); - if (descriptorSize < 32) descriptorSize = 32; + if (descriptorSize < 64 || descriptorSize > blockSize || (descriptorSize & 7) != 0) + throw new InvalidDataException($"Lustre/ldiskfs: invalid 64-bit group descriptor size {descriptorSize}."); } - if (descriptorSize > blockSize) - throw new InvalidDataException($"Lustre/ldiskfs: group descriptor size {descriptorSize} exceeds block size {blockSize}."); var groupCount64 = (totalBlocks - firstDataBlock + blocksPerGroup - 1) / blocksPerGroup; if (groupCount64 is 0 or > uint.MaxValue) @@ -226,10 +231,14 @@ private static Geometry ReadGeometry(Stream image) { return new Geometry( blockSize, totalBlocks, firstDataBlock, blocksPerGroup, groupCount, - descriptorSize, descriptorTableOffset, featureCompat, featureIncompat, featureRoCompat); + descriptorSize, descriptorTableOffset, featureCompat, featureIncompat, featureRoCompat, + checksumSeed); } private static void ValidateWritableProfile(Geometry geometry) { + if ((geometry.FeatureRoCompat & RoCompatMetadataCsum) == 0) + throw new NotSupportedException( + "Lustre destructive maintenance requires ldiskfs metadata_csum so corrupted allocation bitmaps can be detected before any block is overwritten."); if ((geometry.FeatureIncompat & IncompatRecover) != 0) throw new NotSupportedException("Lustre maintenance refuses an ldiskfs image that still needs journal recovery; replay it first."); if ((geometry.FeatureIncompat & IncompatJournalDev) != 0) @@ -249,12 +258,17 @@ private static void ValidateShrinkProfile(Stream image, Geometry geometry) { throw new NotSupportedException("Lustre shrink does not yet rewrite SPARSE_SUPER2 backup locations."); var descriptor = new byte[geometry.DescriptorSize]; + var bitmap = new byte[geometry.BlockSize]; for (uint group = 0; group < geometry.GroupCount; ++group) { ReadDescriptor(image, geometry, group, descriptor); + ValidateGroupDescriptorChecksum(geometry, group, descriptor); + var flags = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.AsSpan(0x12, 2)); if ((flags & BgBlockUninit) != 0) throw new NotSupportedException( $"Lustre shrink refuses lazy BLOCK_UNINIT group {group}; its on-disk allocation bitmap is not authoritative yet."); + + ReadAndValidateBlockBitmap(image, geometry, group, descriptor, bitmap); } } @@ -266,6 +280,25 @@ private static void ReadDescriptor(Stream image, Geometry geometry, uint group, image.ReadExactly(destination); } + private static void ReadAndValidateBlockBitmap( + Stream image, + Geometry geometry, + uint group, + ReadOnlySpan descriptor, + byte[] bitmap) { + var bitmapBlock = ReadBlockBitmapBlock(descriptor, geometry.DescriptorSize); + if (bitmapBlock >= geometry.TotalBlocks) + throw new InvalidDataException($"Lustre/ldiskfs: group {group} block bitmap points outside the volume ({bitmapBlock})."); + + var bitmapOffset = CheckedByteOffset(bitmapBlock, geometry.BlockSize); + if (bitmapOffset + geometry.BlockSize > image.Length) + throw new InvalidDataException($"Lustre/ldiskfs: group {group} block bitmap is truncated."); + + image.Position = bitmapOffset; + image.ReadExactly(bitmap); + ValidateBlockBitmapChecksum(geometry, group, descriptor, bitmap); + } + private static ulong ReadBlockBitmapBlock(ReadOnlySpan descriptor, int descriptorSize) { ulong block = BinaryPrimitives.ReadUInt32LittleEndian(descriptor[..4]); if (descriptorSize >= 64) @@ -273,6 +306,60 @@ private static ulong ReadBlockBitmapBlock(ReadOnlySpan descriptor, int des return block; } + private static void ValidateSuperblockChecksum(ReadOnlySpan superblock) { + var expected = BinaryPrimitives.ReadUInt32LittleEndian(superblock.Slice(0x3FC, 4)); + var actual = Crc32c(0xFFFFFFFFu, superblock[..0x3FC]); + if (actual != expected) + throw new InvalidDataException( + $"Lustre/ldiskfs: superblock metadata checksum mismatch (stored 0x{expected:X8}, calculated 0x{actual:X8})."); + } + + private static void ValidateGroupDescriptorChecksum(Geometry geometry, uint group, byte[] descriptor) { + var expected = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.AsSpan(0x1E, 2)); + var copy = (byte[])descriptor.Clone(); + BinaryPrimitives.WriteUInt16LittleEndian(copy.AsSpan(0x1E, 2), 0); + + Span groupBytes = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(groupBytes, group); + var actual = Crc32c(Crc32c(geometry.ChecksumSeed, groupBytes), copy); + if ((ushort)actual != expected) + throw new InvalidDataException( + $"Lustre/ldiskfs: group {group} descriptor checksum mismatch (stored 0x{expected:X4}, calculated 0x{(ushort)actual:X4})."); + } + + private static void ValidateBlockBitmapChecksum( + Geometry geometry, + uint group, + ReadOnlySpan descriptor, + ReadOnlySpan bitmap) { + var meaningfulBytes = checked((int)(((ulong)geometry.BlocksPerGroup + 7) / 8)); + if (meaningfulBytes > bitmap.Length) + throw new InvalidDataException($"Lustre/ldiskfs: group {group} block bitmap geometry exceeds its bitmap block."); + + var actual = Crc32c(geometry.ChecksumSeed, bitmap[..meaningfulBytes]); + uint expected = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.Slice(0x18, 2)); + if (geometry.DescriptorSize >= 64) + expected |= (uint)BinaryPrimitives.ReadUInt16LittleEndian(descriptor.Slice(0x38, 2)) << 16; + + var matches = geometry.DescriptorSize >= 64 + ? actual == expected + : (ushort)actual == (ushort)expected; + if (!matches) + throw new InvalidDataException( + $"Lustre/ldiskfs: group {group} block bitmap checksum mismatch (stored 0x{expected:X8}, calculated 0x{actual:X8})."); + } + + private static uint Crc32c(uint seed, ReadOnlySpan data) { + const uint polynomial = 0x82F63B78u; + var crc = seed; + foreach (var value in data) { + crc ^= value; + for (var bit = 0; bit < 8; ++bit) + crc = (crc & 1) != 0 ? (crc >> 1) ^ polynomial : crc >> 1; + } + return crc; + } + private static long CheckedByteOffset(ulong block, int blockSize) { if (block > (ulong)(long.MaxValue / blockSize)) throw new InvalidDataException("Lustre/ldiskfs: block address exceeds stream addressing range."); From 825c7233c12232ab217191ed7bb2ed47815d2d5a Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:17:39 +0200 Subject: [PATCH 124/165] Keep BLOCK_UNINIT fail-closed before checksum validation --- FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs b/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs index 9c0d60d43..a9964adb1 100644 --- a/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs +++ b/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs @@ -70,12 +70,15 @@ public static long WipeFreeBlocks(Stream image) { for (uint group = 0; group < geometry.GroupCount; ++group) { ReadDescriptor(image, geometry, group, descriptor); - ValidateGroupDescriptorChecksum(geometry, group, descriptor); + // BLOCK_UNINIT is itself a safe stop condition: if corruption merely sets this + // bit we skip a writable group rather than touching it. If corruption clears the + // bit on a genuinely uninitialised group, the descriptor checksum below catches it. var flags = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.AsSpan(0x12, 2)); if ((flags & BgBlockUninit) != 0) continue; + ValidateGroupDescriptorChecksum(geometry, group, descriptor); ReadAndValidateBlockBitmap(image, geometry, group, descriptor, bitmap); var groupFirst = (ulong)geometry.FirstDataBlock + (ulong)group * geometry.BlocksPerGroup; @@ -261,13 +264,12 @@ private static void ValidateShrinkProfile(Stream image, Geometry geometry) { var bitmap = new byte[geometry.BlockSize]; for (uint group = 0; group < geometry.GroupCount; ++group) { ReadDescriptor(image, geometry, group, descriptor); - ValidateGroupDescriptorChecksum(geometry, group, descriptor); - var flags = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.AsSpan(0x12, 2)); if ((flags & BgBlockUninit) != 0) throw new NotSupportedException( $"Lustre shrink refuses lazy BLOCK_UNINIT group {group}; its on-disk allocation bitmap is not authoritative yet."); + ValidateGroupDescriptorChecksum(geometry, group, descriptor); ReadAndValidateBlockBitmap(image, geometry, group, descriptor, bitmap); } } From 391194be71b08db39857d4f291ec773d47a7af5c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:18:00 +0200 Subject: [PATCH 125/165] Cover Lustre allocation checksum failures --- .../Lustre/LustreMaintenanceSafetyTests.cs | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 Compression.Tests/Lustre/LustreMaintenanceSafetyTests.cs diff --git a/Compression.Tests/Lustre/LustreMaintenanceSafetyTests.cs b/Compression.Tests/Lustre/LustreMaintenanceSafetyTests.cs new file mode 100644 index 000000000..a801e606e --- /dev/null +++ b/Compression.Tests/Lustre/LustreMaintenanceSafetyTests.cs @@ -0,0 +1,71 @@ +using System.Buffers.Binary; +using Compression.Registry; +using FileSystem.Ext; +using FileSystem.Lustre; + +namespace Compression.Tests.Lustre; + +[TestFixture] +public sealed class LustreMaintenanceSafetyTests { + + [Test, Category("ExceptionalCase")] + public void Wipe_RefusesCorruptedBlockBitmapBeforeWritingAnything() { + var data = BuildChecksummedLdiskfsImage(); + CorruptFirstBlockBitmap(data); + var expected = (byte[])data.Clone(); + + using var image = new MemoryStream(data, writable: true); + var descriptor = new LustreFormatDescriptor(); + + var ex = Assert.Throws(() => ((IWipeEmpty)descriptor).WipeUnusedSpace(image)); + Assert.That(ex!.Message, Does.Contain("bitmap checksum mismatch").IgnoreCase); + Assert.That(image.ToArray(), Is.EqualTo(expected), + "Checksum failure must happen before the wipe overwrites any block."); + } + + [Test, Category("ExceptionalCase")] + public void Shrink_RefusesCorruptedBlockBitmapWithoutTouchingOutput() { + var data = BuildChecksummedLdiskfsImage(); + CorruptFirstBlockBitmap(data); + + using var source = new MemoryStream(data, writable: false); + using var target = new MemoryStream([0x51, 0x52, 0x53, 0x54], writable: true); + var descriptor = new LustreFormatDescriptor(); + + var ex = Assert.Throws(() => ((IArchiveShrinkable)descriptor).Shrink(source, target)); + Assert.That(ex!.Message, Does.Contain("bitmap checksum mismatch").IgnoreCase); + Assert.That(target.ToArray(), Is.EqualTo(new byte[] { 0x51, 0x52, 0x53, 0x54 }), + "Shrink preflight must validate source allocation metadata before replacing output."); + } + + private static byte[] BuildChecksummedLdiskfsImage() { + var writer = new ExtWriter(); + writer.AddFile("OBJECTS/0_1", Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray()); + return writer.Build( + blockSize: 4096, + totalBlocks: 4096, + version: ExtWriter.ExtVersion.Ext4, + journal: true, + volumeLabel: "lustre-OST0000", + inodeSize: 256); + } + + private static void CorruptFirstBlockBitmap(byte[] image) { + var superblock = image.AsSpan(1024, 1024); + var blockSize = 1024 << (int)BinaryPrimitives.ReadUInt32LittleEndian(superblock.Slice(24, 4)); + var firstDataBlock = BinaryPrimitives.ReadUInt32LittleEndian(superblock.Slice(20, 4)); + var featureIncompat = BinaryPrimitives.ReadUInt32LittleEndian(superblock.Slice(96, 4)); + var descriptorSize = (featureIncompat & 0x0080) != 0 + ? BinaryPrimitives.ReadUInt16LittleEndian(superblock.Slice(0xFE, 2)) + : 32; + var descriptorOffset = checked((int)((firstDataBlock + 1L) * blockSize)); + var descriptor = image.AsSpan(descriptorOffset, descriptorSize); + + ulong bitmapBlock = BinaryPrimitives.ReadUInt32LittleEndian(descriptor[..4]); + if (descriptorSize >= 64) + bitmapBlock |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(descriptor.Slice(0x20, 4)) << 32; + + var bitmapOffset = checked((int)(bitmapBlock * (ulong)blockSize)); + image[bitmapOffset] ^= 0x01; + } +} From 6af714bdfc9f4bf581ab7057b6558a85e4c6f485 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:19:42 +0200 Subject: [PATCH 126/165] + expose documented OneFS disk geometry --- FileSystems/FileSystem.OneFs/OneFsReader.cs | 57 ++++++++++++--------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/FileSystems/FileSystem.OneFs/OneFsReader.cs b/FileSystems/FileSystem.OneFs/OneFsReader.cs index 303ad4e7e..1a8db2384 100644 --- a/FileSystems/FileSystem.OneFs/OneFsReader.cs +++ b/FileSystems/FileSystem.OneFs/OneFsReader.cs @@ -12,18 +12,28 @@ namespace FileSystem.OneFs; /// /// /// Dell documents the cluster-level structures required to resolve OneFS data: -/// the superblock points at the LIN master, the LIN B+ tree maps logical inode -/// numbers to mirrored inode addresses, and file metatrees map logical blocks to -/// protection groups. Those addresses identify a node, drive and physical block, -/// so a lone drive image is not a self-contained filesystem namespace. +/// superblocks live at multiple fixed block addresses on every drive and point +/// at the LIN master, the LIN B+ tree maps logical inode numbers to mirrored +/// inode addresses, and file metatrees map logical blocks to protection groups. +/// Those addresses identify a node, drive and physical block, so a lone drive +/// image is not a self-contained filesystem namespace. +/// +/// +/// Dell also documents enough physical geometry for safe analysis: every data +/// disk is divided into 32 MiB cylinder groups made of 8 KiB filesystem blocks, +/// with a per-cylinder-group bitmap tracking whether blocks are used for data, +/// inodes or other metadata. The bitmap's raw serialization and location are not +/// publicly specified, so this reader reports the geometry but does not attempt +/// to interpret allocation state. /// /// /// Dell does not publish a fixed raw-image magic value that identifies a OneFS /// drive at offset zero. In particular, the historical "OneFS" / "ONEF" /// literals previously used here could not be corroborated by Dell documentation -/// and are therefore not parsed or advertised as signatures. The reader is -/// reached by explicit format selection or the .onefs extension and treats -/// the supplied bytes as opaque media. +/// and are therefore not parsed or advertised as signatures. Public OneFS logs +/// do show that a superblock magic is validated, but neither its authoritative +/// value nor the fixed superblock block addresses have been found in a public +/// byte-level specification. /// /// /// The reader deliberately performs no payload reads while listing. This keeps a @@ -31,24 +41,18 @@ namespace FileSystem.OneFs; /// inventing structure from undocumented bytes. The raw image remains available /// as a bounded streaming entry for forensic inspection or export. /// -/// -/// References used for the clean-room behaviour documented here: -/// -/// Dell Technologies Info Hub, "OneFS Metadata" (LIN tree, -/// inode mirrors, IFM/DFM B+ trees and protection groups). -/// Dell PowerScale OneFS Technical Overview, "File system -/// structure" (distributed UFS-based filesystem and single namespace). -/// Dell PowerScale OneFS Technical Specifications Guide -/// (8 KiB filesystem block size). -/// -/// No external implementation code is copied or translated. -/// /// public sealed class OneFsReader : IDisposable { /// Documented OneFS filesystem block size. public const int PhysicalBlockSize = 8 * 1024; + /// Documented physical cylinder-group size of each OneFS data disk. + public const int CylinderGroupSize = 32 * 1024 * 1024; + + /// Documented number of 8 KiB blocks in a 32 MiB cylinder group. + public const int BlocksPerCylinderGroup = CylinderGroupSize / PhysicalBlockSize; + /// Name of the synthetic inspection metadata entry. public const string MetadataEntryName = "metadata.ini"; @@ -135,17 +139,24 @@ private byte[] BuildMetadata() { builder.Append("format=Dell PowerScale / Isilon OneFS\n"); builder.Append("detection=extension-or-explicit-selection\n"); builder.Append("authoritative_raw_magic=not_published\n"); + builder.Append("superblock_magic=known-to-exist-value-not-publicly-verified\n"); + builder.Append("superblock_locations=multiple-fixed-block-addresses-values-not-publicly-verified\n"); + builder.Append("superblock_role=references-LIN-master\n"); builder.Append("physical_block_size=").Append(PhysicalBlockSize.ToString(CultureInfo.InvariantCulture)).Append('\n'); + builder.Append("cylinder_group_size=").Append(CylinderGroupSize.ToString(CultureInfo.InvariantCulture)).Append('\n'); + builder.Append("blocks_per_cylinder_group=").Append(BlocksPerCylinderGroup.ToString(CultureInfo.InvariantCulture)).Append('\n'); + builder.Append("allocation_tracking=per-cylinder-group-bitmap-serialization-not-published\n"); builder.Append("image_size=").Append(this.ImageSize.ToString(CultureInfo.InvariantCulture)).Append('\n'); builder.Append("rw_promotion=blocked\n"); builder.Append("rw_promotion_reason_1=OneFS exposes one namespace across the cluster, not one self-contained namespace per drive\n"); builder.Append("rw_promotion_reason_2=LIN B+ tree entries resolve logical inode numbers to mirrored inode addresses on node+drive+block tuples\n"); builder.Append("rw_promotion_reason_3=IFM metatrees resolve logical file blocks to protection groups distributed across cluster nodes and drives\n"); - builder.Append("rw_promotion_reason_4=no published raw-media serialization/update specification or offline single-drive checker was found\n"); + builder.Append("rw_promotion_reason_4=safe writes use distributed two-phase commit and per-node journals\n"); + builder.Append("rw_promotion_reason_5=no published byte-level allocation/tree/journal serialization or offline single-drive checker was found\n"); builder.Append("maintenance=blocked\n"); - builder.Append("maintenance_reason=free/live allocation, relocation metadata, protection-group membership and transaction rules cannot be proven from one opaque image\n"); - builder.Append("ufs_note=Dell describes OneFS as UFS-based; that architectural ancestry does not make an isolated /ifs drive image a generic standalone UFS volume\n"); - builder.Append("note=The raw image is exposed byte-for-byte for inspection only; no undocumented bytes are interpreted or rewritten.\n"); + builder.Append("maintenance_reason=bitmap location and encoding, relocation metadata, protection-group membership and transaction rules cannot be proven from one opaque image\n"); + builder.Append("ufs_note=OneFS is FreeBSD-derived and early Isilon material describes BAM as working with or instead of BSD UFS; an isolated /ifs data drive is not established as a generic standalone UFS volume\n"); + builder.Append("note=The raw image is exposed byte-for-byte for inspection only; only documented geometry is interpreted.\n"); return Encoding.UTF8.GetBytes(builder.ToString()); } From 776e90f8ae4f9894705ce0cd4f098a2a59273a0a Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:20:16 +0200 Subject: [PATCH 127/165] + add OneFS analysis-only layout geometry --- .../FileSystem.OneFs/OneFsFormatDescriptor.cs | 68 ++++++++++++++----- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs b/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs index 5ab7ff6f5..1ea9ea4a2 100644 --- a/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs +++ b/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs @@ -16,6 +16,14 @@ namespace FileSystem.OneFs; /// to reconstruct arbitrary files or safely rewrite their allocation metadata. /// /// +/// The public architecture does expose stable physical geometry: data disks are +/// divided into 32 MiB cylinder groups containing 8 KiB filesystem blocks, with +/// an allocation bitmap per group. This descriptor reports that geometry through +/// but deliberately inherits the interface's +/// refusing rebuild default. Analysis is therefore available without advertising +/// the repository's Layout or Compact maintenance verbs. +/// +/// /// No Dell publication found during the implementation review defines the /// historical "OneFS" or "ONEF" literals as fixed offset-zero raw /// media signatures. Advertising those bytes caused unrelated files beginning @@ -24,26 +32,29 @@ namespace FileSystem.OneFs; /// /// /// R/W and the destructive maintenance verbs remain deliberately unavailable. -/// Implementing add/remove, purge, wipe, relocation/defrag, shrink or relayout -/// without the raw allocation, protection and transaction formats would corrupt -/// media while satisfying only this library's own synthetic tests. The reader -/// instead exposes metadata describing the known architecture plus the original -/// image as a byte-exact bounded stream. +/// Safe OneFS writes use distributed two-phase commit and per-node journals; +/// modifying allocation or tree state on one isolated drive would bypass that +/// protocol. The reader instead exposes documented geometry and architecture plus +/// the original image as a byte-exact bounded stream. /// /// /// Clean-room references: /// /// https://infohub.delltechnologies.com/en-us/p/onefs-metadata/ /// — LIN tree, inode mirrors, IFM/DFM B+ trees and protection metadata. -/// https://infohub.delltechnologies.com/en-nz/l/dell-powerscale-onefs-technical-overview/file-system-structure/1/ -/// — distributed UFS-based filesystem and global namespace. -/// Dell PowerScale OneFS Technical Specifications Guide — -/// fixed 8 KiB filesystem block size. +/// Dell, "High Availability and Data Protection with Dell +/// PowerScale Scale-Out NAS" — 32 MiB cylinder groups, 8 KiB blocks, +/// per-group allocation bitmaps and BAM/LBM architecture. +/// Dell Info Hub, OneFS data inlining / data reduction — +/// fixed-address superblocks and the superblock → LIN-master chain. +/// Isilon patents US8214400B2 and US7937421B2 — distributed +/// mirrored index-tree and BAM/LBM behavioural model. Patent material is used +/// only as an implementation-independent behavioural oracle. /// /// No external implementation code is copied or translated. /// /// -public sealed class OneFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { +public sealed class OneFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, ILayoutOptimizable { /// Gets the registry id. public string Id => "OneFs"; @@ -84,12 +95,13 @@ public sealed class OneFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOpe /// Gets a description of the intentionally limited inspection surface. public string Description => - "Dell PowerScale / Isilon OneFS — opaque single-image inspection only. " + - "OneFS presents a cluster-wide namespace: LIN metadata resolves to mirrored node/drive/block addresses " + - "and file metatrees resolve data through distributed protection groups. Dell documents OneFS as UFS-based, " + - "but does not publish a standalone raw-drive layout or fixed offset-zero media magic sufficient for offline parsing. " + - "The historic 'OneFS'/'ONEF' signature claim was therefore removed. R/W, purge, wipe, defrag, shrink and relayout " + - "remain blocked until the raw allocation/protection/transaction format can be proven against a real cluster or checker."; + "Dell PowerScale / Isilon OneFS — conservative single-image inspection with documented geometry analysis. " + + "OneFS data disks use 8 KiB blocks in 32 MiB cylinder groups, while the namespace and file metatrees resolve " + + "through mirrored node/drive/block addresses and distributed protection groups. Fixed-address superblocks point " + + "toward the LIN master, but Dell does not publish the byte-level superblock/tree/allocation serialization or an " + + "authoritative offset-zero media magic. The historic 'OneFS'/'ONEF' signature claim was therefore removed. " + + "R/W, purge, wipe, defrag, shrink and layout rebuild remain blocked until allocation/protection/journal updates can " + + "be proven against a real cluster or independent checker."; /// Lists the two conservative inspection entries without reading the image payload. public List List(Stream stream, string? password) { @@ -123,4 +135,28 @@ Stream IArchiveFormatOperations.OpenEntry(Stream archive, string entryName, stri ?? throw new FileNotFoundException($"OneFS entry not found: {entryName}", entryName); return reader.OpenEntry(entry); } + + /// + /// Reports the fixed physical geometry documented by Dell without reading or + /// interpreting proprietary allocation metadata. + /// + public LayoutAnalysis AnalyzeLayout(Stream image) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanSeek) + throw new ArgumentException("OneFS layout analysis requires a seekable stream.", nameof(image)); + + return new LayoutAnalysis { + ImageSize = image.Length, + CurrentUnitSize = OneFsReader.PhysicalBlockSize, + CurrentSlackBytes = 0, + OptimalUnitSize = OneFsReader.PhysicalBlockSize, + OptimalSlackBytes = 0, + Notes = [ + $"Documented fixed filesystem block size: {OneFsReader.PhysicalBlockSize} bytes.", + $"Documented cylinder-group size: {OneFsReader.CylinderGroupSize} bytes ({OneFsReader.BlocksPerCylinderGroup} blocks).", + "Each cylinder group has an allocation bitmap, but its raw location/encoding is not public; slack/free-space savings are therefore unknown and intentionally reported as zero.", + "Analysis only: rebuilding or patching OneFS geometry is unsupported and the Layout/Compact maintenance verbs remain unavailable.", + ], + }; + } } From d82df30829f5b62a55b0b00dbf2965a084789c4d Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:20:49 +0200 Subject: [PATCH 128/165] + test OneFS documented geometry analysis --- .../OneFs/OneFsDetectionTests.cs | 49 +++++++++++++++++-- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/Compression.Tests/OneFs/OneFsDetectionTests.cs b/Compression.Tests/OneFs/OneFsDetectionTests.cs index e2b5b29b0..5ed3d2f9c 100644 --- a/Compression.Tests/OneFs/OneFsDetectionTests.cs +++ b/Compression.Tests/OneFs/OneFsDetectionTests.cs @@ -1,6 +1,7 @@ using System.Text; using Compression.Registry; using Compression.Registry.Streaming; +using Compression.Tests.Documentation; using FileSystem.OneFs; namespace Compression.Tests.OneFs; @@ -8,8 +9,8 @@ namespace Compression.Tests.OneFs; /// /// Acceptance tests for the conservative OneFS single-image inspection surface. /// The tests intentionally do not invent an on-disk filesystem image: Dell's -/// public material documents cluster architecture, not a standalone raw-drive -/// serialization that this suite could author independently. +/// public material documents cluster architecture and physical geometry, not a +/// complete standalone raw-drive serialization that this suite could author. /// [TestFixture] public class OneFsDetectionTests { @@ -83,15 +84,52 @@ public void Metadata_ReportsOnlyDocumentedArchitecture() { Assert.That(text, Does.Contain("parse_status=opaque-single-image")); Assert.That(text, Does.Contain("stage=0")); Assert.That(text, Does.Contain("authoritative_raw_magic=not_published")); + Assert.That(text, Does.Contain("superblock_magic=known-to-exist-value-not-publicly-verified")); + Assert.That(text, Does.Contain("superblock_locations=multiple-fixed-block-addresses-values-not-publicly-verified")); + Assert.That(text, Does.Contain("superblock_role=references-LIN-master")); Assert.That(text, Does.Contain($"physical_block_size={OneFsReader.PhysicalBlockSize}")); + Assert.That(text, Does.Contain($"cylinder_group_size={OneFsReader.CylinderGroupSize}")); + Assert.That(text, Does.Contain($"blocks_per_cylinder_group={OneFsReader.BlocksPerCylinderGroup}")); + Assert.That(text, Does.Contain("allocation_tracking=per-cylinder-group-bitmap-serialization-not-published")); Assert.That(text, Does.Contain("LIN B+ tree")); Assert.That(text, Does.Contain("protection groups")); + Assert.That(text, Does.Contain("two-phase commit")); Assert.That(text, Does.Contain("rw_promotion=blocked")); Assert.That(text, Does.Contain("maintenance=blocked")); - Assert.That(text, Does.Contain("UFS-based")); + Assert.That(text, Does.Contain("FreeBSD-derived")); }); } + [Test, Category("HappyPath")] + public void LayoutAnalysis_ReportsDocumentedFixedGeometryWithoutReadingImage() { + var descriptor = new OneFsFormatDescriptor(); + using var image = new MemoryStream(BuildOpaqueImage(4096)); + image.Position = 73; + + var layout = (ILayoutOptimizable)descriptor; + var analysis = layout.AnalyzeLayout(image); + + Assert.Multiple(() => { + Assert.That(analysis.ImageSize, Is.EqualTo(image.Length)); + Assert.That(analysis.CurrentUnitSize, Is.EqualTo(OneFsReader.PhysicalBlockSize)); + Assert.That(analysis.OptimalUnitSize, Is.EqualTo(OneFsReader.PhysicalBlockSize)); + Assert.That(analysis.PotentialSavingsBytes, Is.Zero, + "Unknown allocation/slack must not be turned into invented savings."); + Assert.That(analysis.Notes, Has.Some.Contains("33554432")); + Assert.That(analysis.Notes, Has.Some.Contains("4096 blocks")); + Assert.That(image.Position, Is.EqualTo(73), + "Geometry analysis is documentary and must not scan the opaque image."); + Assert.That(FilesystemSupportMatrix.RelaysOut(descriptor), Is.False, + "Analysis-only ILayoutOptimizable must not advertise a working Layout rebuild."); + }); + + using var target = new MemoryStream(); + Assert.That( + () => layout.RebuildStreaming(image, target, new LayoutRebuildOptions()), + Throws.TypeOf(), + "No creator exists, so the default layout rebuild must fail closed."); + } + [Test, Category("Regression")] public void LegacyMagicLookingBytes_AreTreatedAsOpaquePayload() { var imageBytes = BuildOpaqueImage(256); @@ -112,7 +150,7 @@ public void LegacyMagicLookingBytes_AreTreatedAsOpaquePayload() { } [Test, Category("Stub")] - public void WriteAndMaintenanceCapabilities_RemainBlocked() { + public void WriteAndDestructiveMaintenanceCapabilities_RemainBlocked() { var descriptor = new OneFsFormatDescriptor(); Assert.Multiple(() => { @@ -123,7 +161,8 @@ public void WriteAndMaintenanceCapabilities_RemainBlocked() { Assert.That(descriptor, Is.Not.InstanceOf()); Assert.That(descriptor, Is.Not.InstanceOf()); Assert.That(descriptor, Is.Not.InstanceOf()); - Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf(), + "Documented fixed geometry is safe to analyze even though rewriting it is not."); Assert.That(descriptor, Is.Not.InstanceOf()); }); } From 28b25c5d31605311ffa5002b5409b7a52dc94eaa Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:21:56 +0200 Subject: [PATCH 129/165] docs(onefs): record deeper raw-media research --- FileSystems/FileSystem.OneFs/FORMAT-NOTES.md | 215 ++++++++++++++----- 1 file changed, 164 insertions(+), 51 deletions(-) diff --git a/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md b/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md index 3a5de17a5..28ff5d161 100644 --- a/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md +++ b/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md @@ -6,80 +6,193 @@ running PowerScale cluster and must not silently substitute OneFS REST, SMB, NFS or SSH operations for the repository's `Stream`-based image contracts. -The implementation review in September 2026 found enough public information to -model the architecture, but not enough to reconstruct or mutate a standalone -OneFS drive image safely. +The September 2026 review found substantially more public structure than the +original Stage-0 stub assumed, but still not enough byte-level serialization to +reconstruct or mutate a standalone OneFS data drive safely. -## Publicly documented facts +## What is publicly established -Dell's published material establishes the following relevant invariants: +Dell's own architecture material establishes these invariants: - OneFS presents a **single global namespace across the cluster**. -- Files and directories are identified by LINs (logical inode numbers). -- The LIN B+ tree maps LINs to **mirrored inode disk addresses**. Dell's own - examples identify a node, drive number, physical block address and inode size. -- File IFM/metatrees map logical block numbers to **protection groups**; directory +- Files and directories are identified by 64-bit LINs (logical inode numbers). +- The **superblock exists at multiple fixed block addresses on every drive**. + It contains addresses for the LIN master, which leads to the root of the LIN + B+ tree. Dell publishes this traversal concept but not the fixed block numbers + or byte layout of the superblock. +- The LIN B+ tree maps LINs to **mirrored inode disk addresses**. `isi get` + examples render an inode address as a node, logical drive, byte/block address + and inode size, for example `1,9,1054720:512`. +- File IFM/metatrees map logical file blocks into **protection groups**; directory DFM trees hold names and namespace relationships. -- Inodes and B+ tree blocks are mirrored; the LIN root is especially heavily - mirrored. -- File data is divided into 8 KiB filesystem blocks and distributed across the - cluster according to protection/layout policy. -- Dell describes OneFS as UFS-based, but the data filesystem is distributed - across nodes. UFS ancestry is therefore not evidence that a single `/ifs` - drive can be parsed or rewritten as a normal standalone UFS volume. - -## What could not be established - -No Dell publication or independently verifiable implementation found during the -review defined enough of the raw-media serialization to support safe offline -mutation. In particular, the review did **not** establish: +- Data addresses shown by `isi get -DD` use the same distributed shape and can + identify sparse regions as `0,0,0:8192`. +- Each data disk is assigned a GUID and logical drive number and is divided into + **32 MiB cylinder groups** consisting of **8 KiB filesystem blocks**. That is + 4096 filesystem blocks per cylinder group. +- Every cylinder group tracks, with a bitmap, whether blocks are used for file + data, inodes, or other metadata. Public material does not define the bitmap's + raw offset or serialization. +- Layout is managed by BAM (Block Allocation Manager); local disk allocation is + delegated to the LBM (Local Block Manager), while remote block operations go + through the RBM. +- Safe writes are distributed transactions. BAM Safe Write builds a write plan, + then OneFS uses two-phase commit across participating nodes and their journals. + A correct offline writer therefore cannot update a single drive in isolation + and pretend cluster consistency was preserved. +- OneFS is based on UFS / FreeBSD heritage, but its `/ifs` storage layer replaces + or extends the ordinary standalone-UFS allocation model with distributed + BAM/LBM/RBM addressing and protection. UFS ancestry is useful historical + context, not a license to route an isolated OneFS data disk through `UfsReader`. + +## `newfs_efs` and the UFS-looking format logs + +A deeper search found old reimage logs where `newfs_efs` prints classic BSD-newfs +style geometry such as an 8 KiB block size, 1 KiB fragments, cylinder groups and +backup-superblock locations. That specific log formats `/dev/imdd5a`, however: +IMDD is OneFS's **Mirrored Device Driver for system volumes** such as root/`/var`, +not proof that an `/ifs` data disk is a normal UFS volume. + +Separately, Dell's reimage documentation says `isi_initial_newfs` then formats the +`/ifs` partition on all drives and announces `Proceeding with v16 newfs...` while +creating a linked journal. Combined with Dell's documented 32 MiB/8 KiB data-disk +geometry, this confirms substantial BSD/UFS lineage in the formatter and physical +organization, but it still does not publish the proprietary `/ifs` superblock, +allocation bitmap, LIN tree or transaction record layouts. + +The practical conclusion is narrower than the old stub's wording: **OneFS is +UFS-derived, but an isolated OneFS data drive is not established as a complete, +generic UFS filesystem image.** + +## Patent-level structural evidence + +Several Isilon patents predate Dell's acquisition and describe implementation +behavior in enough detail to sharpen the model without supplying a usable raw +serialization: + +- US8214400B2 / US7797283 describes a distributed mirrored index tree. A + superblock contains a header plus block-address (`baddr`) pointers to copies of + the tree root. The header includes version/tree-height information; unused + pointers may be zero. Inner nodes keep sorted keys/offsets at one end and + mirrored child pointers at the other, leaving free space between them. +- US7937421B2 / application 20040153479 describes BAM layout selection and names + `ifs_lbn_t`, `ifs_devid_t` and `ifs_baddr_t`; disk/block allocation itself is + delegated to LBM. +- US7743033 describes BAM as working with and/or instead of BSD UFS, which is + consistent with Dell's later description of OneFS as UFS-based while using a + distributed allocation layer. + +These patents are used only as behavioral/specification evidence. Their +pseudocode and expressive implementation are **not copied or translated**. + +## Other behavioral oracles found + +Public Dell command output gives useful consistency checks for a future reader: + +- `isi get -D` exposes inode version, directory version, inode revision, mirror + count, parent LIN/hash, physical-block counts and rendered IFS inode addresses. +- `isi get -DD` exposes protection groups and distributed physical data addresses. +- Dell documents OneFS NFS file-handle chunks as little-endian; a file handle can + carry the 64-bit LIN. This is evidence about an exported protocol encoding, not + enough to assume every raw-disk integer uses the same encoding. +- OneFS release/recovery logs contain `Bad Magic in superblock`, proving a + superblock magic check exists in at least relevant OneFS metadata/journal paths. + The expected value and authoritative raw-drive offset remain unverified. +- Commercial recovery reports describe scanning **all drive images** for OneFS + inode and directory-entry copies, then combining those results cluster-wide. + That independently reinforces the multi-drive reconstruction requirement. + +A public GitHub repository named `jji0717/cp-migrate` contains source that refers +to proprietary OneFS headers such as `ifs/ifs_types.h`, `ifs/ifm/ifm_dinode.h` +and BAM APIs. The repository declares **no license** and does not contain the +missing proprietary header definitions. It is therefore treated only as a source +of behavioral/symbol-name clues; no code, layout or expressive structure from it +is reused here. + +## What remains unknown at byte level + +No authoritative publication or independently licensed implementation found in +the review defines enough raw serialization for safe traversal or mutation. In +particular, the review still does **not** establish: - a fixed offset-zero `"OneFS"` or `"ONEF"` drive signature; -- the complete superblock/LIN-master serialization and version rules; -- inode and B+ tree record encodings across supported OneFS generations; -- free-space/allocation metadata sufficient to distinguish proven-dead blocks; -- protection-group membership and parity/update rules sufficient to relocate or - replace file data; -- journal/transaction ordering required to leave an edited cluster consistent; +- the actual superblock magic, exact fixed superblock block addresses, checksum, + header size, field offsets or version compatibility rules; +- the LIN-master record serialization; +- inode and B+/B* tree record encodings across supported OneFS generations; +- the cylinder-group allocation bitmap location, width, bit meanings or update + rules; +- protection-group/parity record serialization needed to reconstruct or relocate + file content; +- journal and two-phase-commit record formats needed to make crash-consistent + writes; - an offline single-drive checker that can act as an independent write oracle. -The previous descriptor's `"OneFS"` / `"ONEF"` magic therefore had no acceptable -reference and has been removed rather than promoted into more code. +The previous descriptor's `"OneFS"` / `"ONEF"` offset-zero magic therefore still +has no acceptable reference and remains removed. ## Capability decision | Capability | State | Reason | | --- | --- | --- | | list / extract | limited | synthetic metadata + byte-exact raw-image stream only | +| geometry analysis | **supported** | Dell fixes blocks at 8 KiB and groups at 32 MiB; no payload parsing required | | create | blocked | no public raw-media writer specification/oracle | -| modify / R/W | blocked | namespace, allocation and protection state are cluster-wide | -| purge | blocked | cannot remove live namespace entries and update protection metadata safely | -| wipe | blocked | cannot prove which omitted/raw blocks are unused; guessing is destructive | -| defrag | blocked | relocation requires allocation + IFM/protection-group updates | -| shrink | blocked | no proven tail/free-space ownership or canonical smaller geometry | -| layout | blocked | no proven creator/rebuilder for alternate geometry | -| compact | blocked | composite of defrag/layout/shrink, all blocked above | - -This is deliberately stricter than implementing a no-op maintenance interface: -the support matrix treats an advertised interface as a working verb. Reporting a -successful wipe that merely writes zero bytes would be technically callable but -operationally dishonest. +| modify / R/W | blocked | namespace, allocation, protection and journal state are cluster-wide | +| purge | blocked | cannot remove live namespace entries and atomically update distributed metadata | +| wipe | blocked | bitmap location/encoding is unknown, so no raw block can be proven free safely | +| defrag | blocked | relocation requires IFM/protection-group/allocation updates and distributed commit | +| shrink | blocked | no proven tail ownership or supported smaller physical geometry | +| layout rebuild | blocked | fixed geometry is known; a valid alternate creator/rebuilder is not | +| compact | blocked | composite of defrag/layout/shrink, all rewriting steps remain blocked | + +`OneFsFormatDescriptor` therefore implements `ILayoutOptimizable` **only for +analysis**. `AnalyzeLayout` reports the documented 8 KiB unit and 32 MiB cylinder +group. It does not estimate unknown slack and does not override `RebuildStreaming`. +The support matrix consequently keeps Layout and Compact unadvertised. + +This is deliberately stricter than implementing no-op maintenance interfaces: +the support matrix treats an advertised rewriting interface as a working verb. +Returning success while changing nothing—or guessing that silent regions are +free—would be operationally dishonest and potentially destructive. ## References +Primary / official material: + - Dell Technologies Info Hub, **OneFS Metadata**: https://infohub.delltechnologies.com/en-us/p/onefs-metadata/ +- Dell Technologies Info Hub, **OneFS architectural overview** (32 MiB cylinder + groups, 8 KiB blocks, allocation bitmaps, BAM/LBM/RBM, safe writes): + https://infohub.delltechnologies.com/en-us/l/high-availability-and-data-protection-with-dell-powerscale-scale-out-nas/onefs-architectural-overview-1/1/ +- Dell Technologies Info Hub, **OneFS data inlining / performance** (fixed-address + superblocks and LIN-master traversal): + https://infohub.delltechnologies.com/en-us/l/dell-powerscale-onefs-data-reduction-and-storage-efficiency/performance-854/ - Dell PowerScale OneFS Technical Overview, **File system structure**: https://infohub.delltechnologies.com/en-nz/l/dell-powerscale-onefs-technical-overview/file-system-structure/1/ -- Dell PowerScale OneFS 9.15 Administration Guide, **Structure of the file system**: - https://www.dell.com/support/manuals/en-us/isilon-onefs/ifs_pub_91500_administration_guide_gui/structure-of-the-file-system -- Dell PowerScale OneFS Technical Specifications Guide, **File system guidelines** - (8 KiB block size): - https://www.dell.com/support/manuals/en-us/isilon-onefs/ifs_pub_onefs_tech_spec_guide_9.9.0.0/file-system-guidelines +- Dell PowerScale reimage procedure, `isi_initial_newfs` / `/ifs` formatting + behavior (Dell-hosted/support material; mirrored copies may require support + access depending on region/account). +- Dell KB, **How to obtain a LIN from an NFS file handle** (little-endian exported + LIN encoding): + https://www.dell.com/support/kbdoc/en-us/000019498/isilon-onefs-how-to-obtain-a-lin-from-an-nfs-file-handle + +Implementation-independent patent evidence: + +- US8214400B2, **Systems and methods for maintaining distributed data**: + https://patents.google.com/patent/US8214400B2/en +- US7937421B2, **Systems and methods for restriping files in a distributed file + system**: + https://patents.google.com/patent/US7937421B2/en +- US7743033, **Systems and methods for providing a distributed file system + utilizing metadata to track information about data stored throughout the + system**. ## Licensing / implementation provenance -The sources above are used only for factual behavior and architecture. No Dell -implementation source, comments, naming, control flow or other expressive code -was copied or translated. The implementation is an original clean-room C# -inspection surface under this repository's license. +Dell documentation and patents are used only to determine factual architecture, +public behavior, constants/geometry and acceptance boundaries. Patent pseudocode +is not copied or translated. The unlicensed `cp-migrate` repository is not a code +source. No external implementation code, comments, naming structure or control +flow is reproduced. The implementation remains original managed C# under this +repository's license, with no new dependency. From 4953a32eb10c65dde04cabffdceafd1ad4b4f7c8 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:22:41 +0200 Subject: [PATCH 130/165] Add ext xattr read and inline mutation support --- .../FileSystem.Ext/ExtExtendedAttributes.cs | 403 ++++++++++++++++++ 1 file changed, 403 insertions(+) create mode 100644 FileSystems/FileSystem.Ext/ExtExtendedAttributes.cs diff --git a/FileSystems/FileSystem.Ext/ExtExtendedAttributes.cs b/FileSystems/FileSystem.Ext/ExtExtendedAttributes.cs new file mode 100644 index 000000000..904a58f0c --- /dev/null +++ b/FileSystems/FileSystem.Ext/ExtExtendedAttributes.cs @@ -0,0 +1,403 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; +using Compression.Core.DiskImage; + +namespace FileSystem.Ext; + +/// +/// Reads ext2/3/4 extended attributes and performs conservative in-inode mutations. +/// +/// +/// +/// ext stores small xattrs after the extended inode fields and may store additional +/// attributes in one external xattr block referenced by i_file_acl. Both forms +/// are readable here. Mutations deliberately target only the in-inode form: growing +/// or rewriting an external xattr block also requires block allocation/reference-count +/// and metadata-checksum publication, so unsupported cases fail closed. +/// +/// +/// The implementation follows the Linux ext4 on-disk documentation. No kernel +/// implementation code is copied. +/// +/// +public static class ExtExtendedAttributes { + private const int SuperblockOffset = 1024; + private const ushort ExtMagic = 0xEF53; + private const uint XattrMagic = 0xEA020000; + private const uint Incompat64Bit = 0x0080; + private const uint IncompatCsumSeed = 0x2000; + private const uint RoCompatMetadataCsum = 0x0400; + private const int ClassicInodeSize = 128; + private const int EntryHeaderSize = 16; + + /// Reads all directly stored xattrs for . + public static IReadOnlyDictionary Read(Stream image, string path) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + EnsureReadableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inodeOffset = geometry.InodeOffset(image, inodeNumber); + var inode = ReadAt(image, inodeOffset, geometry.InodeSize); + var result = new Dictionary(StringComparer.Ordinal); + + ReadInline(inode, result); + + var externalBlock = ExternalXattrBlock(inode, geometry.Has64Bit); + if (externalBlock != 0) { + var blockOffset = checked((long)externalBlock * geometry.BlockSize); + if (blockOffset < 0 || blockOffset + geometry.BlockSize > image.Length) + throw new InvalidDataException("ext: external xattr block lies outside the image."); + var block = ReadAt(image, blockOffset, geometry.BlockSize); + if (BinaryPrimitives.ReadUInt32LittleEndian(block) != XattrMagic) + throw new InvalidDataException("ext: i_file_acl does not reference an ext xattr block."); + ReadEntries(block, EntryHeaderOffset: 32, ValueBaseOffset: 0, result); + } + + return result; + } finally { + image.Position = original; + } + } + + /// + /// Creates or replaces one in-inode xattr. External-block and EA-inode values + /// are never rewritten by this conservative mutator. + /// + public static void Set(Stream image, string path, string name, ReadOnlySpan value) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + EnsureWritableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inodeOffset = geometry.InodeOffset(image, inodeNumber); + var inode = ReadAt(image, inodeOffset, geometry.InodeSize); + + var externalBlock = ExternalXattrBlock(inode, geometry.Has64Bit); + if (externalBlock != 0) { + var external = ReadExternal(image, geometry, externalBlock); + if (external.ContainsKey(name)) + throw new NotSupportedException( + $"ext: xattr '{name}' is stored in an external xattr block; external-block mutation is intentionally unsupported."); + } + + var attributes = ReadInlineDictionary(inode); + attributes[name] = value.ToArray(); + WriteInline(inode, attributes); + StampInodeIfNeeded(geometry, inodeNumber, inode); + WriteAt(image, inodeOffset, inode); + image.Flush(); + } finally { + image.Position = original; + } + } + + /// Removes one in-inode xattr, returning false when it was absent. + public static bool Remove(Stream image, string path, string name) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + EnsureWritableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inodeOffset = geometry.InodeOffset(image, inodeNumber); + var inode = ReadAt(image, inodeOffset, geometry.InodeSize); + + var attributes = ReadInlineDictionary(inode); + if (attributes.Remove(name)) { + WriteInline(inode, attributes); + StampInodeIfNeeded(geometry, inodeNumber, inode); + WriteAt(image, inodeOffset, inode); + image.Flush(); + return true; + } + + var externalBlock = ExternalXattrBlock(inode, geometry.Has64Bit); + if (externalBlock != 0 && ReadExternal(image, geometry, externalBlock).ContainsKey(name)) + throw new NotSupportedException( + $"ext: xattr '{name}' is stored in an external xattr block; external-block mutation is intentionally unsupported."); + + return false; + } finally { + image.Position = original; + } + } + + private static Dictionary ReadInlineDictionary(byte[] inode) { + var result = new Dictionary(StringComparer.Ordinal); + ReadInline(inode, result); + return result; + } + + private static void ReadInline(byte[] inode, Dictionary result) { + if (inode.Length <= ClassicInodeSize) return; + var extraIsize = BinaryPrimitives.ReadUInt16LittleEndian(inode.AsSpan(ClassicInodeSize)); + var headerOffset = ClassicInodeSize + extraIsize; + if (headerOffset < ClassicInodeSize || headerOffset + 4 > inode.Length) return; + if (BinaryPrimitives.ReadUInt32LittleEndian(inode.AsSpan(headerOffset)) != XattrMagic) return; + + // For inode-body xattrs e_value_offs is relative to the first xattr entry, + // immediately after the four-byte ibody header. + var firstEntry = headerOffset + 4; + ReadEntries(inode, firstEntry, firstEntry, result); + } + + private static Dictionary ReadExternal(Stream image, Geometry geometry, ulong blockNumber) { + var offset = checked((long)blockNumber * geometry.BlockSize); + if (offset < 0 || offset + geometry.BlockSize > image.Length) + throw new InvalidDataException("ext: external xattr block lies outside the image."); + var block = ReadAt(image, offset, geometry.BlockSize); + if (BinaryPrimitives.ReadUInt32LittleEndian(block) != XattrMagic) + throw new InvalidDataException("ext: invalid external xattr block magic."); + var result = new Dictionary(StringComparer.Ordinal); + ReadEntries(block, 32, 0, result); + return result; + } + + private static void ReadEntries( + byte[] storage, + int EntryHeaderOffset, + int ValueBaseOffset, + Dictionary result) { + var pos = EntryHeaderOffset; + while (pos + EntryHeaderSize <= storage.Length) { + // Four zero bytes terminate the entry array. + if (BinaryPrimitives.ReadUInt32LittleEndian(storage.AsSpan(pos)) == 0) break; + + var nameLength = storage[pos]; + var nameIndex = storage[pos + 1]; + var valueOffset = BinaryPrimitives.ReadUInt16LittleEndian(storage.AsSpan(pos + 2)); + var valueInode = BinaryPrimitives.ReadUInt32LittleEndian(storage.AsSpan(pos + 4)); + var valueSize = BinaryPrimitives.ReadUInt32LittleEndian(storage.AsSpan(pos + 8)); + var entryLength = Align4(EntryHeaderSize + nameLength); + if (nameLength == 0 || pos + entryLength > storage.Length) + throw new InvalidDataException("ext: malformed xattr entry."); + if (valueInode != 0) + throw new NotSupportedException("ext: EA-inode xattr values are not supported by the image accessor."); + if (valueSize > int.MaxValue) + throw new InvalidDataException("ext: xattr value is too large."); + + var valueAt = checked(ValueBaseOffset + valueOffset); + if (valueAt < 0 || valueAt + (long)valueSize > storage.Length) + throw new InvalidDataException("ext: xattr value lies outside its storage region."); + + var suffix = Encoding.UTF8.GetString(storage, pos + EntryHeaderSize, nameLength); + var fullName = ExpandName(nameIndex, suffix); + var value = storage.AsSpan(valueAt, (int)valueSize).ToArray(); + result[fullName] = value; + pos += entryLength; + } + } + + private static void WriteInline(byte[] inode, IReadOnlyDictionary attributes) { + if (inode.Length <= ClassicInodeSize) + throw new NotSupportedException("ext: 128-byte inodes have no in-inode xattr area."); + + var extraIsize = BinaryPrimitives.ReadUInt16LittleEndian(inode.AsSpan(ClassicInodeSize)); + if (extraIsize == 0) + throw new NotSupportedException("ext: inode does not declare an extended inode area for inline xattrs."); + + var headerOffset = ClassicInodeSize + extraIsize; + if (headerOffset < ClassicInodeSize || headerOffset + 4 > inode.Length) + throw new InvalidDataException("ext: invalid i_extra_isize leaves no xattr body header."); + + var region = inode.AsSpan(headerOffset); + region.Clear(); + if (attributes.Count == 0) return; + + BinaryPrimitives.WriteUInt32LittleEndian(region, XattrMagic); + var firstEntry = headerOffset + 4; + var entryPos = firstEntry; + var valuePos = inode.Length; + + foreach (var pair in attributes.OrderBy(pair => pair.Key, StringComparer.Ordinal)) { + var (nameIndex, suffix) = CompressName(pair.Key); + var nameBytes = Encoding.UTF8.GetBytes(suffix); + if (nameBytes.Length is 0 or > byte.MaxValue) + throw new ArgumentException($"ext: xattr name '{pair.Key}' is not representable."); + var value = pair.Value ?? []; + + valuePos = AlignDown4(valuePos - value.Length); + var entryLength = Align4(EntryHeaderSize + nameBytes.Length); + if (entryPos + entryLength + 4 > valuePos) + throw new NotSupportedException( + $"ext: xattrs do not fit in inode {inode.Length}-byte inline storage; external-block allocation is not implemented."); + + var relativeValueOffset = valuePos - firstEntry; + if ((uint)relativeValueOffset > ushort.MaxValue) + throw new NotSupportedException("ext: inline xattr value offset exceeds the on-disk field width."); + + inode[entryPos] = (byte)nameBytes.Length; + inode[entryPos + 1] = nameIndex; + BinaryPrimitives.WriteUInt16LittleEndian(inode.AsSpan(entryPos + 2), (ushort)relativeValueOffset); + BinaryPrimitives.WriteUInt32LittleEndian(inode.AsSpan(entryPos + 4), 0); // e_value_inum + BinaryPrimitives.WriteUInt32LittleEndian(inode.AsSpan(entryPos + 8), checked((uint)value.Length)); + BinaryPrimitives.WriteUInt32LittleEndian(inode.AsSpan(entryPos + 12), 0); // e_hash optional for ibody attrs + nameBytes.CopyTo(inode.AsSpan(entryPos + EntryHeaderSize)); + value.CopyTo(inode.AsSpan(valuePos)); + entryPos += entryLength; + } + + // Four zero bytes after the final entry terminate the entry array; the region + // was cleared before encoding, so no additional write is needed. + } + + private static ulong ExternalXattrBlock(ReadOnlySpan inode, bool has64Bit) { + if (inode.Length < 108) return 0; + var low = BinaryPrimitives.ReadUInt32LittleEndian(inode[0x68..]); + ulong high = 0; + // Linux ext4 inode osd2 starts at 0x74; l_i_file_acl_high is the u16 at 0x76. + if (has64Bit && inode.Length >= 0x78) + high = BinaryPrimitives.ReadUInt16LittleEndian(inode[0x76..]); + return low | high << 32; + } + + private static string ExpandName(byte index, string suffix) => index switch { + 1 => "user." + suffix, + 2 => "system.posix_acl_access" + suffix, + 3 => "system.posix_acl_default" + suffix, + 4 => "trusted." + suffix, + 6 => "security." + suffix, + 7 => "system." + suffix, + _ => $"ext.index{index}." + suffix, + }; + + private static (byte Index, string Suffix) CompressName(string name) { + if (name.StartsWith("user.", StringComparison.Ordinal)) return (1, name[5..]); + if (name.Equals("system.posix_acl_access", StringComparison.Ordinal)) return (2, ""); + if (name.Equals("system.posix_acl_default", StringComparison.Ordinal)) return (3, ""); + if (name.StartsWith("trusted.", StringComparison.Ordinal)) return (4, name[8..]); + if (name.StartsWith("security.", StringComparison.Ordinal)) return (6, name[9..]); + if (name.StartsWith("system.", StringComparison.Ordinal)) return (7, name[7..]); + throw new ArgumentException( + $"ext: xattr '{name}' has no supported namespace prefix (user., trusted., security., system.).", + nameof(name)); + } + + private static uint ResolveInode(Stream image, string path, uint rootInode) { + var normalized = Normalize(path); + if (normalized.Length == 0) return rootInode; + image.Position = 0; + using var reader = new ExtReader(image, leaveOpen: true); + var entry = reader.Entries.FirstOrDefault(entry => + string.Equals(Normalize(entry.Name), normalized, StringComparison.Ordinal)); + return entry?.Inode ?? throw new FileNotFoundException($"ext entry not found: {path}"); + } + + private static string Normalize(string path) => path.Replace('\\', '/').Trim('/'); + + private static void StampInodeIfNeeded(Geometry geometry, uint inodeNumber, byte[] inode) { + if (!geometry.HasMetadataChecksum) return; + var generation = inode.Length >= 0x68 + ? BinaryPrimitives.ReadUInt32LittleEndian(inode.AsSpan(0x64)) + : 0; + var inodeSeed = ExtChecksums.InodeSeed(geometry.ChecksumSeed, inodeNumber, generation); + ExtChecksums.StampInode(inode, geometry.InodeSize, inodeSeed); + } + + private static byte[] ReadAt(Stream image, long offset, int length) { + var data = new byte[length]; + image.Position = offset; + image.ReadExactly(data); + return data; + } + + private static void WriteAt(Stream image, long offset, ReadOnlySpan data) { + image.Position = offset; + image.Write(data); + } + + private static int Align4(int value) => checked((value + 3) & ~3); + private static int AlignDown4(int value) => value & ~3; + + private static void EnsureReadableSeekable(Stream image) { + if (!image.CanRead || !image.CanSeek) + throw new ArgumentException("ext xattr access requires a readable, seekable image.", nameof(image)); + } + + private static void EnsureWritableSeekable(Stream image) { + EnsureReadableSeekable(image); + if (!image.CanWrite) + throw new ArgumentException("ext xattr mutation requires a writable image.", nameof(image)); + } + + private readonly record struct Geometry( + int BlockSize, + uint InodesPerGroup, + int InodeSize, + uint FirstDataBlock, + int DescriptorSize, + bool Has64Bit, + bool HasMetadataChecksum, + uint ChecksumSeed, + uint RootInode) { + + public static Geometry Read(Stream image) { + if (image.Length < SuperblockOffset + 1024) + throw new InvalidDataException("ext: image is too small for its superblock."); + var sb = ReadAt(image, SuperblockOffset, 1024); + if (BinaryPrimitives.ReadUInt16LittleEndian(sb.AsSpan(56)) != ExtMagic) + throw new InvalidDataException("ext: invalid superblock magic."); + + var blockSize = checked(1024 << (int)BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(24))); + var inodesPerGroup = BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(40)); + var inodeSize = BinaryPrimitives.ReadUInt16LittleEndian(sb.AsSpan(88)); + if (inodeSize == 0) inodeSize = ClassicInodeSize; + var firstDataBlock = BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(20)); + var featureIncompat = BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(96)); + var featureRoCompat = BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(100)); + var descriptorSize = ExtBlockGroupGeometry.DescriptorSize(sb); + var checksumSeed = (featureIncompat & IncompatCsumSeed) != 0 + ? BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(624)) + : ExtChecksums.SeedFromUuid(sb.AsSpan(104, 16)); + + if (blockSize is < 1024 or > 65536 || (blockSize & (blockSize - 1)) != 0) + throw new InvalidDataException($"ext: invalid block size {blockSize}."); + if (inodesPerGroup == 0) + throw new InvalidDataException("ext: superblock declares zero inodes per group."); + if (inodeSize < ClassicInodeSize) + throw new InvalidDataException($"ext: invalid inode size {inodeSize}."); + + return new Geometry( + blockSize, + inodesPerGroup, + inodeSize, + firstDataBlock, + descriptorSize, + (featureIncompat & Incompat64Bit) != 0, + (featureRoCompat & RoCompatMetadataCsum) != 0, + checksumSeed, + RootInode: 2); + } + + public long InodeOffset(Stream image, uint inodeNumber) { + if (inodeNumber == 0) + throw new ArgumentOutOfRangeException(nameof(inodeNumber)); + var group = (inodeNumber - 1) / this.InodesPerGroup; + var index = (inodeNumber - 1) % this.InodesPerGroup; + var bgdtBlock = this.FirstDataBlock + 1; + var descriptorOffset = checked((long)bgdtBlock * this.BlockSize + (long)group * this.DescriptorSize); + if (descriptorOffset + this.DescriptorSize > image.Length) + throw new InvalidDataException("ext: inode group descriptor lies outside the image."); + var descriptor = ReadAt(image, descriptorOffset, this.DescriptorSize); + ulong tableBlock = BinaryPrimitives.ReadUInt32LittleEndian(descriptor.AsSpan(8)); + if (this.Has64Bit && descriptor.Length >= 44) + tableBlock |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(descriptor.AsSpan(40)) << 32; + var offset = checked((long)tableBlock * this.BlockSize + (long)index * this.InodeSize); + if (offset < 0 || offset + this.InodeSize > image.Length) + throw new InvalidDataException("ext: inode lies outside the image."); + return offset; + } + } +} From b7e474062dd8c903fc55c17d9f123c0e287e9289 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:23:56 +0200 Subject: [PATCH 131/165] Add XFS short-form xattr mutation support --- .../FileSystem.Xfs/XfsExtendedAttributes.cs | 388 ++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100644 FileSystems/FileSystem.Xfs/XfsExtendedAttributes.cs diff --git a/FileSystems/FileSystem.Xfs/XfsExtendedAttributes.cs b/FileSystems/FileSystem.Xfs/XfsExtendedAttributes.cs new file mode 100644 index 000000000..2e96a4ffc --- /dev/null +++ b/FileSystems/FileSystem.Xfs/XfsExtendedAttributes.cs @@ -0,0 +1,388 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; +using Compression.Core.Checksums; + +namespace FileSystem.Xfs; + +/// +/// Reads and mutates XFS short-form extended attributes stored in an inode's +/// attribute fork. +/// +/// +/// +/// XFS can promote an attribute fork from short-form to leaf/node btrees and can +/// place large values in remote blocks. This accessor intentionally stops before +/// that boundary: local inode attributes are fully round-trippable, while leaf, +/// node, and remote-value mutation fails closed instead of rewriting structures +/// the current workbench does not yet allocate transactionally. +/// +/// +/// Layout and namespace constants are taken from the published XFS on-disk +/// specification; no Linux implementation code is copied. +/// +/// +public static class XfsExtendedAttributes { + private const uint XfsMagic = 0x58465342; // XFSB + private const ushort InodeMagic = 0x494E; // IN + private const byte FormatLocal = 1; + private const byte FormatExtents = 2; + private const byte FormatBtree = 3; + private const byte AttrRoot = 0x02; + private const byte AttrSecure = 0x04; + private const byte AttrParent = 0x08; + private const byte AttrIncomplete = 0x80; + private const int DiCrcOffset = 100; + private const int ShortFormHeaderSize = 4; // be16 totsize, u8 count, u8 padding + private const int ShortFormEntryHeaderSize = 3; // namelen, valuelen, flags + + /// Reads all short-form xattrs for . + public static IReadOnlyDictionary Read(Stream image, string path) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + EnsureReadableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inode = ReadAt(image, geometry.InodeOffset(inodeNumber), geometry.InodeSize); + return ParseShortForm(inode, geometry).ToDictionary( + attribute => ExpandName(attribute.Flags, attribute.Name), + attribute => attribute.Value, + StringComparer.Ordinal); + } finally { + image.Position = original; + } + } + + /// + /// Creates or replaces a short-form xattr. If the resulting fork does not fit + /// in the inode, the call fails without modifying the image. + /// + public static void Set(Stream image, string path, string name, ReadOnlySpan value) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + EnsureWritableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inodeOffset = geometry.InodeOffset(inodeNumber); + var inode = ReadAt(image, inodeOffset, geometry.InodeSize); + ValidateInode(inode); + + var attributes = ParseShortForm(inode, geometry).ToList(); + var (namespaceFlags, suffix) = CompressName(name); + var suffixBytes = Encoding.UTF8.GetBytes(suffix); + if (suffixBytes.Length is 0 or > byte.MaxValue) + throw new ArgumentException($"XFS xattr name '{name}' is not representable in short-form storage.", nameof(name)); + if (value.Length > byte.MaxValue) + throw new NotSupportedException("XFS short-form xattr values are limited to 255 bytes; remote-value allocation is not implemented."); + + var index = attributes.FindIndex(attribute => + SameNamespace(attribute.Flags, namespaceFlags) && + string.Equals(attribute.Name, suffix, StringComparison.Ordinal)); + if (index >= 0) { + var existing = attributes[index]; + attributes[index] = existing with { Value = value.ToArray() }; + } else { + attributes.Add(new ShortFormAttribute(suffix, value.ToArray(), namespaceFlags)); + } + + WriteShortForm(inode, geometry, attributes); + StampInodeCrcIfNeeded(inode, geometry); + WriteAt(image, inodeOffset, inode); + image.Flush(); + } finally { + image.Position = original; + } + } + + /// Removes one short-form xattr, returning false when it was absent. + public static bool Remove(Stream image, string path, string name) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + EnsureWritableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inodeOffset = geometry.InodeOffset(inodeNumber); + var inode = ReadAt(image, inodeOffset, geometry.InodeSize); + ValidateInode(inode); + + var attributes = ParseShortForm(inode, geometry).ToList(); + var (namespaceFlags, suffix) = CompressName(name); + var removed = attributes.RemoveAll(attribute => + SameNamespace(attribute.Flags, namespaceFlags) && + string.Equals(attribute.Name, suffix, StringComparison.Ordinal)) > 0; + if (!removed) return false; + + WriteShortForm(inode, geometry, attributes); + StampInodeCrcIfNeeded(inode, geometry); + WriteAt(image, inodeOffset, inode); + image.Flush(); + return true; + } finally { + image.Position = original; + } + } + + private static IReadOnlyList ParseShortForm(byte[] inode, Geometry geometry) { + ValidateInode(inode); + var forkoff = inode[82]; + if (forkoff == 0) return []; + + var attrFormat = inode[83]; + if (attrFormat != FormatLocal) + throw new NotSupportedException( + $"XFS: attribute fork format {attrFormat} is not short-form; leaf/btree xattr access is not implemented."); + + var start = checked(geometry.CoreSize + forkoff * 8); + if (start + ShortFormHeaderSize > inode.Length) + throw new InvalidDataException("XFS: short-form attribute fork begins outside the inode."); + + var totalSize = BinaryPrimitives.ReadUInt16BigEndian(inode.AsSpan(start)); + var count = inode[start + 2]; + if (totalSize < ShortFormHeaderSize || start + totalSize > inode.Length) + throw new InvalidDataException("XFS: invalid short-form xattr size."); + + var result = new List(count); + var pos = start + ShortFormHeaderSize; + var end = start + totalSize; + for (var i = 0; i < count; ++i) { + if (pos + ShortFormEntryHeaderSize > end) + throw new InvalidDataException("XFS: truncated short-form xattr entry."); + var nameLength = inode[pos]; + var valueLength = inode[pos + 1]; + var flags = inode[pos + 2]; + var recordLength = ShortFormEntryHeaderSize + nameLength + valueLength; + if (nameLength == 0 || pos + recordLength > end) + throw new InvalidDataException("XFS: malformed short-form xattr entry."); + + var name = Encoding.UTF8.GetString(inode, pos + ShortFormEntryHeaderSize, nameLength); + var value = inode.AsSpan(pos + ShortFormEntryHeaderSize + nameLength, valueLength).ToArray(); + result.Add(new ShortFormAttribute(name, value, flags)); + pos += recordLength; + } + + return result; + } + + private static void WriteShortForm( + byte[] inode, + Geometry geometry, + IReadOnlyList attributes) { + ValidateInode(inode); + + foreach (var attribute in attributes) { + if ((attribute.Flags & (AttrParent | AttrIncomplete)) != 0) + throw new NotSupportedException( + "XFS: parent-pointer or incomplete xattrs cannot be rewritten by the short-form accessor."); + } + + if (attributes.Count == 0) { + var oldForkoff = inode[82]; + if (oldForkoff != 0) { + var oldStart = checked(geometry.CoreSize + oldForkoff * 8); + if (oldStart < inode.Length) inode.AsSpan(oldStart).Clear(); + } + BinaryPrimitives.WriteUInt16BigEndian(inode.AsSpan(80), 0); // di_anextents + inode[82] = 0; // di_forkoff + inode[83] = 0; // di_aformat + return; + } + + if (attributes.Count > byte.MaxValue) + throw new NotSupportedException("XFS short-form xattrs cannot encode more than 255 attributes."); + + var encodedSize = ShortFormHeaderSize; + foreach (var attribute in attributes) { + var nameLength = Encoding.UTF8.GetByteCount(attribute.Name); + if (nameLength is 0 or > byte.MaxValue || attribute.Value.Length > byte.MaxValue) + throw new NotSupportedException("XFS xattr cannot be represented in short-form storage."); + encodedSize = checked(encodedSize + ShortFormEntryHeaderSize + nameLength + attribute.Value.Length); + } + + var existingForkoff = inode[82]; + int forkoff; + if (existingForkoff != 0) { + if (inode[83] != FormatLocal) + throw new NotSupportedException("XFS: existing non-local attribute fork cannot be rewritten as short-form."); + forkoff = existingForkoff; + } else { + var dataBytes = DataForkBytesUsed(inode, geometry); + forkoff = checked((dataBytes + 7) / 8); + if (forkoff == 0) forkoff = 1; + } + + if (forkoff > byte.MaxValue) + throw new NotSupportedException("XFS: data fork leaves no representable attribute-fork offset."); + var start = checked(geometry.CoreSize + forkoff * 8); + if (start + encodedSize > inode.Length) + throw new NotSupportedException( + "XFS: xattrs do not fit in the inode attribute fork; leaf/remote attribute allocation is not implemented."); + + // Build off-image first so a capacity/encoding failure cannot leave a half-written fork. + var encoded = new byte[encodedSize]; + BinaryPrimitives.WriteUInt16BigEndian(encoded, checked((ushort)encodedSize)); + encoded[2] = checked((byte)attributes.Count); + encoded[3] = 0; + var pos = ShortFormHeaderSize; + foreach (var attribute in attributes) { + var nameBytes = Encoding.UTF8.GetBytes(attribute.Name); + encoded[pos] = checked((byte)nameBytes.Length); + encoded[pos + 1] = checked((byte)attribute.Value.Length); + encoded[pos + 2] = attribute.Flags; + nameBytes.AsSpan().CopyTo(encoded.AsSpan(pos + ShortFormEntryHeaderSize)); + attribute.Value.AsSpan().CopyTo(encoded.AsSpan(pos + ShortFormEntryHeaderSize + nameBytes.Length)); + pos += ShortFormEntryHeaderSize + nameBytes.Length + attribute.Value.Length; + } + + // Preserve the data fork and replace only the attribute-fork region. + inode.AsSpan(start).Clear(); + encoded.AsSpan().CopyTo(inode.AsSpan(start)); + BinaryPrimitives.WriteUInt16BigEndian(inode.AsSpan(80), 0); // short-form has no attr extents + inode[82] = checked((byte)forkoff); + inode[83] = FormatLocal; + } + + private static int DataForkBytesUsed(byte[] inode, Geometry geometry) { + var format = inode[5]; + return format switch { + FormatLocal => checked((int)Math.Min( + BinaryPrimitives.ReadUInt64BigEndian(inode.AsSpan(56)), + (ulong)(inode.Length - geometry.CoreSize))), + FormatExtents => checked((int)BinaryPrimitives.ReadUInt32BigEndian(inode.AsSpan(76)) * 16), + FormatBtree => throw new NotSupportedException( + "XFS: cannot introduce an attr fork beside a btree data fork without decoding its in-inode root size."), + _ => 0, + }; + } + + private static string ExpandName(byte flags, string suffix) { + if ((flags & AttrRoot) != 0) return "trusted." + suffix; + if ((flags & AttrSecure) != 0) return "security." + suffix; + if ((flags & AttrParent) != 0) return "xfs.parent." + suffix; + return "user." + suffix; + } + + private static (byte Flags, string Suffix) CompressName(string name) { + if (name.StartsWith("user.", StringComparison.Ordinal)) return (0, name[5..]); + if (name.StartsWith("trusted.", StringComparison.Ordinal)) return (AttrRoot, name[8..]); + if (name.StartsWith("security.", StringComparison.Ordinal)) return (AttrSecure, name[9..]); + throw new ArgumentException( + $"XFS xattr '{name}' has no supported namespace prefix (user., trusted., security.).", + nameof(name)); + } + + private static bool SameNamespace(byte existingFlags, byte requestedFlags) + => (existingFlags & (AttrRoot | AttrSecure | AttrParent)) == requestedFlags; + + private static ulong ResolveInode(Stream image, string path, ulong rootInode) { + var normalized = Normalize(path); + if (normalized.Length == 0) return rootInode; + image.Position = 0; + using var reader = new XfsReader(image, leaveOpen: true); + var entry = reader.Entries.FirstOrDefault(entry => + string.Equals(Normalize(entry.Name), normalized, StringComparison.Ordinal)); + return entry is null + ? throw new FileNotFoundException($"XFS entry not found: {path}") + : checked((ulong)entry.InodeNumber); + } + + private static string Normalize(string path) => path.Replace('\\', '/').Trim('/'); + + private static void ValidateInode(ReadOnlySpan inode) { + if (inode.Length < 100 || BinaryPrimitives.ReadUInt16BigEndian(inode) != InodeMagic) + throw new InvalidDataException("XFS: invalid inode magic."); + } + + private static void StampInodeCrcIfNeeded(byte[] inode, Geometry geometry) { + if (!geometry.IsV5) return; + inode.AsSpan(DiCrcOffset, sizeof(uint)).Clear(); + var crc = Crc32.Compute(inode, Crc32.Castagnoli); + BinaryPrimitives.WriteUInt32LittleEndian(inode.AsSpan(DiCrcOffset), crc); + } + + private static byte[] ReadAt(Stream image, long offset, int length) { + var data = new byte[length]; + image.Position = offset; + image.ReadExactly(data); + return data; + } + + private static void WriteAt(Stream image, long offset, ReadOnlySpan data) { + image.Position = offset; + image.Write(data); + } + + private static void EnsureReadableSeekable(Stream image) { + if (!image.CanRead || !image.CanSeek) + throw new ArgumentException("XFS xattr access requires a readable, seekable image.", nameof(image)); + } + + private static void EnsureWritableSeekable(Stream image) { + EnsureReadableSeekable(image); + if (!image.CanWrite) + throw new ArgumentException("XFS xattr mutation requires a writable image.", nameof(image)); + } + + private sealed record ShortFormAttribute(string Name, byte[] Value, byte Flags); + + private readonly record struct Geometry( + uint BlockSize, + ushort InodeSize, + ulong RootInode, + uint AgBlocks, + byte AgBlockLog, + int CoreSize, + bool IsV5) { + + public static Geometry Read(Stream image) { + if (image.Length < 512) + throw new InvalidDataException("XFS: image too small for its superblock."); + var sb = ReadAt(image, 0, 512); + if (BinaryPrimitives.ReadUInt32BigEndian(sb) != XfsMagic) + throw new InvalidDataException("XFS: invalid superblock magic."); + + var blockSize = BinaryPrimitives.ReadUInt32BigEndian(sb.AsSpan(4)); + var rootInode = BinaryPrimitives.ReadUInt64BigEndian(sb.AsSpan(56)); + var agBlocks = BinaryPrimitives.ReadUInt32BigEndian(sb.AsSpan(84)); + var version = BinaryPrimitives.ReadUInt16BigEndian(sb.AsSpan(100)); + var inodeSize = BinaryPrimitives.ReadUInt16BigEndian(sb.AsSpan(104)); + var agBlockLog = sb[124]; + if (blockSize == 0) blockSize = 4096; + if (inodeSize == 0) inodeSize = 256; + if (agBlocks == 0) agBlocks = checked((uint)(image.Length / blockSize)); + if (agBlockLog == 0) { + var value = agBlocks; + while (value > 1) { ++agBlockLog; value >>= 1; } + } + + var isV5 = (version & 0xF) >= 5; + return new Geometry(blockSize, inodeSize, rootInode, agBlocks, agBlockLog, + CoreSize: isV5 ? 176 : 100, + IsV5: isV5); + } + + public long InodeOffset(ulong inodeNumber) { + var inodesPerBlock = checked((int)(this.BlockSize / this.InodeSize)); + if (inodesPerBlock <= 0 || (inodesPerBlock & (inodesPerBlock - 1)) != 0) + throw new InvalidDataException("XFS: unsupported inode geometry."); + var inoPbLog = 0; + for (var value = inodesPerBlock; value > 1; value >>= 1) ++inoPbLog; + var aginoLog = this.AgBlockLog + inoPbLog; + var agNumber = inodeNumber >> aginoLog; + var agInode = inodeNumber & ((1UL << aginoLog) - 1); + var block = agInode / (ulong)inodesPerBlock; + var slot = agInode % (ulong)inodesPerBlock; + return checked((long)((agNumber * this.AgBlocks + block) * this.BlockSize + slot * this.InodeSize)); + } + } +} From 9bac5f76f9fe1dc023559a13d6c1d8d57c3fea88 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:24:19 +0200 Subject: [PATCH 132/165] Add multi-session NRG DAO authoring --- FileFormats/FileFormat.Nrg/NrgWriter.cs | 345 ++++++++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 FileFormats/FileFormat.Nrg/NrgWriter.cs diff --git a/FileFormats/FileFormat.Nrg/NrgWriter.cs b/FileFormats/FileFormat.Nrg/NrgWriter.cs new file mode 100644 index 000000000..401c0c334 --- /dev/null +++ b/FileFormats/FileFormat.Nrg/NrgWriter.cs @@ -0,0 +1,345 @@ +using System.Buffers.Binary; +using System.Text; + +namespace FileFormat.Nrg; + +/// NRG v2 track storage modes understood by Nero-compatible readers. +public enum NrgTrackMode : byte { + /// Cooked Mode 1, 2,048 bytes per sector. + Mode1 = 0x00, + /// Cooked Mode 2 Form 1, 2,048 bytes per sector. + Mode2Form1 = 0x02, + /// Mode 2 Form 2 / almost-full sector, 2,336 bytes per sector. + Mode2Form2 = 0x03, + /// Raw Mode 1, 2,352 bytes per sector. + Mode1Raw = 0x05, + /// Raw Mode 2, 2,352 bytes per sector. + Mode2Raw = 0x06, + /// Raw CD-DA audio, 2,352 bytes per sector. + Audio = 0x07, + /// Raw Mode 1 plus 96-byte subchannel, 2,448 bytes per sector. + Mode1RawWithSubchannel = 0x0F, + /// Raw CD-DA audio plus 96-byte subchannel, 2,448 bytes per sector. + AudioWithSubchannel = 0x10, + /// Raw Mode 2 plus 96-byte subchannel, 2,448 bytes per sector. + Mode2RawWithSubchannel = 0x11, +} + +/// One track to be written into an NRG disc image. +/// On-disc/storage mode. +/// Track bytes, already encoded in the sector representation selected by . +public sealed record NrgTrackDefinition(NrgTrackMode Mode, byte[] Data) { + /// Number of sectors in index 00 before index 01. Defaults to no stored pregap. + public int PregapSectors { get; init; } + + /// + /// Optional pregap bytes. When supplied they must contain exactly sectors, + /// or, when is zero, determine the pregap length themselves. + /// + public byte[]? PregapData { get; init; } + + /// Optional 12-character ISRC for this track. + public string? Isrc { get; init; } + + /// Whether the Q-subchannel control byte advertises digital-copy permission. + public bool CopyPermitted { get; init; } +} + +/// One NRG session. Track numbers are assigned globally and consecutively across sessions. +/// Tracks in this session. +public sealed record NrgSessionDefinition(IReadOnlyList Tracks) { + /// Optional 13-digit media catalog number (MCN/EAN-13). + public string? Mcn { get; init; } +} + +/// A complete NRG disc authoring description. +/// Sessions in physical order. +public sealed record NrgDiscDefinition(IReadOnlyList Sessions) { + /// Optional raw CD-TEXT packs. The byte length must be a multiple of 18. + public byte[]? CdText { get; init; } +} + +/// +/// Writes Nero NRG v2 images using the DAO profile (CUEX + DAOX per session). +/// This profile can represent multiple sessions, mixed data/audio tracks, pregaps, MCN, ISRC, +/// raw sector modes and optional 96-byte subchannel data without flattening them into one ISO track. +/// +/// +/// Nero never published a normative NRG specification. The field layout used here is the independently +/// reconstructed public layout also consumed by CDEmu/libMirage and libcdio. No implementation code from +/// those projects is copied; this writer is an independent managed implementation of the observable format. +/// +public static class NrgWriter { + private const uint CdRomMediumType = 0x00000400; + private const int LeadInSectors = 4500; + private const int FirstLeadOutSectors = 6750; + private const int LaterLeadOutSectors = 2250; + + private sealed record TrackLayout( + int TrackNumber, + NrgTrackDefinition Definition, + int SectorSize, + int PregapSectors, + long PregapOffset, + long StartOffset, + long EndOffset, + int Index0Lba, + int Index1Lba); + + private sealed record SessionLayout( + NrgSessionDefinition Definition, + int LeadInLba, + int LeadOutLba, + IReadOnlyList Tracks); + + /// Writes as an NRG v2 image. + public static void Write(Stream output, NrgDiscDefinition disc) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(disc); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("NRG authoring requires a writable, seekable stream.", nameof(output)); + + ValidateDisc(disc); + output.Position = 0; + output.SetLength(0); + + var sessions = WriteTrackData(output, disc); + var trailerOffset = checked((ulong)output.Position); + + foreach (var session in sessions) { + WriteCuex(output, session); + WriteDaox(output, session); + + Span sinf = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(sinf, checked((uint)session.Tracks.Count)); + WriteChunk(output, "SINF"u8, sinf); + } + + if (disc.CdText is { Length: > 0 } cdText) + WriteChunk(output, "CDTX"u8, cdText); + + Span mtyp = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(mtyp, CdRomMediumType); + WriteChunk(output, "MTYP"u8, mtyp); + WriteChunk(output, "END!"u8, ReadOnlySpan.Empty); + + Span footer = stackalloc byte[12]; + "NER5"u8.CopyTo(footer); + BinaryPrimitives.WriteUInt64BigEndian(footer[4..], trailerOffset); + output.Write(footer); + } + + /// Gets the stored sector size implied by an NRG mode code. + public static int GetSectorSize(NrgTrackMode mode) => mode switch { + NrgTrackMode.Mode1 or NrgTrackMode.Mode2Form1 => 2048, + NrgTrackMode.Mode2Form2 => 2336, + NrgTrackMode.Mode1Raw or NrgTrackMode.Mode2Raw or NrgTrackMode.Audio => 2352, + NrgTrackMode.Mode1RawWithSubchannel or NrgTrackMode.AudioWithSubchannel or NrgTrackMode.Mode2RawWithSubchannel => 2448, + _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, "Unsupported NRG track mode."), + }; + + private static List WriteTrackData(Stream output, NrgDiscDefinition disc) { + var sessions = new List(disc.Sessions.Count); + var nextTrackNumber = 1; + var programLba = 0; + var previousLeadOutLba = 0; + + for (var sessionIndex = 0; sessionIndex < disc.Sessions.Count; ++sessionIndex) { + var definition = disc.Sessions[sessionIndex]; + int leadInLba; + if (sessionIndex == 0) { + leadInLba = -150; + programLba = 0; + } else { + var previousLeadOutLength = sessionIndex == 1 ? FirstLeadOutSectors : LaterLeadOutSectors; + leadInLba = checked(previousLeadOutLba + previousLeadOutLength); + programLba = checked(leadInLba + LeadInSectors); + } + + var tracks = new List(definition.Tracks.Count); + foreach (var track in definition.Tracks) { + var sectorSize = GetSectorSize(track.Mode); + var pregapSectors = GetPregapSectors(track, sectorSize); + var pregapOffset = output.Position; + if (pregapSectors != 0) { + if (track.PregapData is { Length: > 0 } pregapData) + output.Write(pregapData); + else + WriteZeros(output, checked((long)pregapSectors * sectorSize)); + } + + var startOffset = output.Position; + output.Write(track.Data); + var endOffset = output.Position; + var index0Lba = programLba; + var index1Lba = checked(index0Lba + pregapSectors); + var dataSectors = track.Data.Length / sectorSize; + programLba = checked(index1Lba + dataSectors); + + tracks.Add(new TrackLayout( + nextTrackNumber++, track, sectorSize, pregapSectors, + pregapOffset, startOffset, endOffset, index0Lba, index1Lba)); + } + + var leadOutLba = programLba; + sessions.Add(new SessionLayout(definition, leadInLba, leadOutLba, tracks)); + previousLeadOutLba = leadOutLba; + } + + return sessions; + } + + private static void WriteCuex(Stream output, SessionLayout session) { + var payload = new byte[checked((session.Tracks.Count * 2 + 2) * 8)]; + var position = 0; + + var firstAdrCtl = AdrCtl(session.Tracks[0].Definition); + WriteCueEntry(payload.AsSpan(position, 8), firstAdrCtl, 0x00, 0x00, session.LeadInLba); + position += 8; + + foreach (var track in session.Tracks) { + var adrCtl = AdrCtl(track.Definition); + var trackBcd = ToBcd(track.TrackNumber); + WriteCueEntry(payload.AsSpan(position, 8), adrCtl, trackBcd, 0x00, track.Index0Lba); + position += 8; + WriteCueEntry(payload.AsSpan(position, 8), adrCtl, trackBcd, 0x01, track.Index1Lba); + position += 8; + } + + var lastAdrCtl = AdrCtl(session.Tracks[^1].Definition); + WriteCueEntry(payload.AsSpan(position, 8), lastAdrCtl, 0xAA, 0x01, session.LeadOutLba); + WriteChunk(output, "CUEX"u8, payload); + } + + private static void WriteDaox(Stream output, SessionLayout session) { + var payload = new byte[checked(22 + session.Tracks.Count * 42)]; + BinaryPrimitives.WriteUInt32BigEndian(payload, checked((uint)payload.Length)); + + if (session.Definition.Mcn is { } mcn) + Encoding.ASCII.GetBytes(mcn, payload.AsSpan(4, 13)); + + payload[17] = 0; + payload[18] = session.Tracks.Any(static track => IsMode2(track.Definition.Mode)) ? (byte)0x20 : (byte)0x00; + payload[19] = 0x01; + payload[20] = checked((byte)session.Tracks[0].TrackNumber); + payload[21] = checked((byte)session.Tracks[^1].TrackNumber); + + var position = 22; + foreach (var track in session.Tracks) { + var record = payload.AsSpan(position, 42); + if (track.Definition.Isrc is { } isrc) + Encoding.ASCII.GetBytes(isrc, record[..12]); + BinaryPrimitives.WriteUInt16BigEndian(record[12..], checked((ushort)track.SectorSize)); + record[14] = (byte)track.Definition.Mode; + record[15] = 0; + BinaryPrimitives.WriteUInt16BigEndian(record[16..], 1); + BinaryPrimitives.WriteUInt64BigEndian(record[18..], checked((ulong)track.PregapOffset)); + BinaryPrimitives.WriteUInt64BigEndian(record[26..], checked((ulong)track.StartOffset)); + BinaryPrimitives.WriteUInt64BigEndian(record[34..], checked((ulong)track.EndOffset)); + position += 42; + } + + WriteChunk(output, "DAOX"u8, payload); + } + + private static void WriteCueEntry(Span destination, byte adrCtl, byte track, byte index, int lba) { + destination[0] = adrCtl; + destination[1] = track; + destination[2] = index; + destination[3] = 0; + BinaryPrimitives.WriteUInt32BigEndian(destination[4..], unchecked((uint)lba)); + } + + private static byte AdrCtl(NrgTrackDefinition track) { + var control = track.Mode is NrgTrackMode.Audio or NrgTrackMode.AudioWithSubchannel ? 0x00 : 0x40; + if (track.CopyPermitted) + control |= 0x20; + return checked((byte)(control | 0x01)); + } + + private static byte ToBcd(int value) { + if (value is < 0 or > 99) + throw new ArgumentOutOfRangeException(nameof(value), value, "BCD track numbers are limited to 0..99."); + return checked((byte)(((value / 10) << 4) | value % 10)); + } + + private static int GetPregapSectors(NrgTrackDefinition track, int sectorSize) { + if (track.PregapSectors < 0) + throw new ArgumentOutOfRangeException(nameof(track.PregapSectors), "Pregap sector count cannot be negative."); + if (track.PregapData is not { Length: > 0 } bytes) + return track.PregapSectors; + if (bytes.Length % sectorSize != 0) + throw new ArgumentException("NRG pregap data must be an integral number of sectors.", nameof(track)); + + var fromBytes = bytes.Length / sectorSize; + if (track.PregapSectors != 0 && track.PregapSectors != fromBytes) + throw new ArgumentException("NRG PregapSectors does not match PregapData length.", nameof(track)); + return fromBytes; + } + + private static void ValidateDisc(NrgDiscDefinition disc) { + if (disc.Sessions is null || disc.Sessions.Count == 0) + throw new ArgumentException("An NRG disc must contain at least one session.", nameof(disc)); + if (disc.CdText is { Length: > 0 } cdText && cdText.Length % 18 != 0) + throw new ArgumentException("NRG CD-TEXT data must contain complete 18-byte packs.", nameof(disc)); + + var trackCount = 0; + foreach (var session in disc.Sessions) { + if (session is null || session.Tracks is null || session.Tracks.Count == 0) + throw new ArgumentException("Every NRG session must contain at least one track.", nameof(disc)); + ValidateMcn(session.Mcn); + + foreach (var track in session.Tracks) { + if (track is null) + throw new ArgumentException("NRG tracks cannot be null.", nameof(disc)); + var sectorSize = GetSectorSize(track.Mode); + if (track.Data is null || track.Data.Length == 0 || track.Data.Length % sectorSize != 0) + throw new ArgumentException($"NRG {track.Mode} track data must be a non-empty multiple of {sectorSize} bytes.", nameof(disc)); + _ = GetPregapSectors(track, sectorSize); + ValidateIsrc(track.Isrc); + } + + trackCount = checked(trackCount + session.Tracks.Count); + } + + if (trackCount > 99) + throw new ArgumentException("Compact Disc NRG images can contain at most 99 tracks.", nameof(disc)); + } + + private static void ValidateMcn(string? mcn) { + if (mcn is null) + return; + if (mcn.Length != 13 || mcn.Any(static c => c is < '0' or > '9')) + throw new ArgumentException("NRG MCN must contain exactly 13 ASCII digits.", nameof(mcn)); + } + + private static void ValidateIsrc(string? isrc) { + if (isrc is null) + return; + if (isrc.Length != 12 || isrc.Any(static c => c > 0x7F || char.IsControl(c))) + throw new ArgumentException("NRG ISRC must contain exactly 12 printable ASCII characters.", nameof(isrc)); + } + + private static bool IsMode2(NrgTrackMode mode) + => mode is NrgTrackMode.Mode2Form1 or NrgTrackMode.Mode2Form2 or NrgTrackMode.Mode2Raw or NrgTrackMode.Mode2RawWithSubchannel; + + private static void WriteZeros(Stream output, long length) { + Span zeros = stackalloc byte[4096]; + while (length > 0) { + var count = (int)Math.Min(length, zeros.Length); + output.Write(zeros[..count]); + length -= count; + } + } + + private static void WriteChunk(Stream output, ReadOnlySpan id, ReadOnlySpan payload) { + if (id.Length != 4) + throw new ArgumentException("NRG chunk ids are exactly four bytes.", nameof(id)); + + Span header = stackalloc byte[8]; + id.CopyTo(header); + BinaryPrimitives.WriteUInt32BigEndian(header[4..], checked((uint)payload.Length)); + output.Write(header); + output.Write(payload); + } +} From af361134a14cea69fe16800bd2b0d8121d1236cb Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:24:24 +0200 Subject: [PATCH 133/165] Test ext xattr round trips --- .../Ext/ExtExtendedAttributesTests.cs | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 Compression.Tests/Ext/ExtExtendedAttributesTests.cs diff --git a/Compression.Tests/Ext/ExtExtendedAttributesTests.cs b/Compression.Tests/Ext/ExtExtendedAttributesTests.cs new file mode 100644 index 000000000..e106935ea --- /dev/null +++ b/Compression.Tests/Ext/ExtExtendedAttributesTests.cs @@ -0,0 +1,70 @@ +using FileSystem.Ext; + +namespace Compression.Tests.Ext; + +[TestFixture] +public class ExtExtendedAttributesTests { + + private static MemoryStream BuildImage() { + var writer = new ExtWriter(); + writer.AddFile("brick/data.txt", "payload"u8.ToArray()); + var bytes = writer.Build( + blockSize: 1024, + totalBlocks: 4096, + version: ExtWriter.ExtVersion.Ext4, + journal: false, + volumeLabel: "xattr-test", + inodeSize: 256); + return new MemoryStream(bytes, writable: true); + } + + [Test, Category("RoundTrip")] + public void TrustedGfid_CreateUpdateRemove_RoundTrips() { + using var image = BuildImage(); + byte[] first = [0x10, 0x32, 0x54, 0x76, 0x98, 0xBA, 0xDC, 0xFE, 1, 2, 3, 4, 5, 6, 7, 8]; + byte[] second = [8, 7, 6, 5, 4, 3, 2, 1, 0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10]; + + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", first); + Assert.That(ExtExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(first)); + + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", second); + Assert.That(ExtExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(second)); + + Assert.That(ExtExtendedAttributes.Remove(image, "brick/data.txt", "trusted.gfid"), Is.True); + Assert.That(ExtExtendedAttributes.Read(image, "brick/data.txt").ContainsKey("trusted.gfid"), Is.False); + Assert.That(ExtExtendedAttributes.Remove(image, "brick/data.txt", "trusted.gfid"), Is.False); + } + + [Test, Category("RoundTrip")] + public void MultipleTrustedAttributes_RoundTripWithoutDisturbingPayload() { + using var image = BuildImage(); + var gfid = Enumerable.Range(0, 16).Select(i => (byte)i).ToArray(); + var dht = Enumerable.Range(16, 16).Select(i => (byte)i).ToArray(); + + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", gfid); + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.glusterfs.dht", dht); + + var attributes = ExtExtendedAttributes.Read(image, "brick/data.txt"); + Assert.Multiple(() => { + Assert.That(attributes["trusted.gfid"], Is.EqualTo(gfid)); + Assert.That(attributes["trusted.glusterfs.dht"], Is.EqualTo(dht)); + }); + + image.Position = 0; + using var reader = new ExtReader(image, leaveOpen: true); + var entry = reader.Entries.Single(entry => entry.Name == "brick/data.txt"); + Assert.That(reader.Extract(entry), Is.EqualTo("payload"u8.ToArray())); + } + + [Test, Category("ErrorHandling")] + public void OversizedInlineSet_FailsWithoutChangingExistingAttribute() { + using var image = BuildImage(); + var gfid = Enumerable.Range(0, 16).Select(i => (byte)i).ToArray(); + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", gfid); + + Assert.Throws(() => + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.too-large", new byte[128])); + + Assert.That(ExtExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(gfid)); + } +} From dad85ecefbc5d0755cdc82955cf301ccf991142b Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:24:38 +0200 Subject: [PATCH 134/165] Test XFS short-form xattr round trips --- .../Xfs/XfsExtendedAttributesTests.cs | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 Compression.Tests/Xfs/XfsExtendedAttributesTests.cs diff --git a/Compression.Tests/Xfs/XfsExtendedAttributesTests.cs b/Compression.Tests/Xfs/XfsExtendedAttributesTests.cs new file mode 100644 index 000000000..fa7a8efd4 --- /dev/null +++ b/Compression.Tests/Xfs/XfsExtendedAttributesTests.cs @@ -0,0 +1,61 @@ +using FileSystem.Xfs; + +namespace Compression.Tests.Xfs; + +[TestFixture] +public class XfsExtendedAttributesTests { + + private static MemoryStream BuildImage() { + var writer = new XfsWriter(); + writer.AddFile("brick/data.txt", "payload"u8.ToArray()); + var image = new MemoryStream(); + writer.WriteTo(image); + image.Position = 0; + return image; + } + + [Test, Category("RoundTrip")] + public void TrustedGfid_CreateUpdateRemove_RoundTrips() { + using var image = BuildImage(); + byte[] first = [0x10, 0x32, 0x54, 0x76, 0x98, 0xBA, 0xDC, 0xFE, 1, 2, 3, 4, 5, 6, 7, 8]; + byte[] second = [8, 7, 6, 5, 4, 3, 2, 1, 0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10]; + + XfsExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", first); + Assert.That(XfsExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(first)); + + XfsExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", second); + Assert.That(XfsExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(second)); + + Assert.That(XfsExtendedAttributes.Remove(image, "brick/data.txt", "trusted.gfid"), Is.True); + Assert.That(XfsExtendedAttributes.Read(image, "brick/data.txt").ContainsKey("trusted.gfid"), Is.False); + Assert.That(XfsExtendedAttributes.Remove(image, "brick/data.txt", "trusted.gfid"), Is.False); + } + + [Test, Category("RoundTrip")] + public void TrustedGfid_PreservesFilePayload() { + using var image = BuildImage(); + var gfid = Enumerable.Range(0, 16).Select(i => (byte)i).ToArray(); + + XfsExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", gfid); + + image.Position = 0; + using var reader = new XfsReader(image, leaveOpen: true); + var entry = reader.Entries.Single(entry => entry.Name == "brick/data.txt"); + Assert.Multiple(() => { + Assert.That(reader.Extract(entry), Is.EqualTo("payload"u8.ToArray())); + Assert.That(XfsExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(gfid)); + }); + } + + [Test, Category("ErrorHandling")] + public void OversizedShortFormSet_FailsWithoutChangingExistingAttribute() { + using var image = BuildImage(); + var gfid = Enumerable.Range(0, 16).Select(i => (byte)i).ToArray(); + XfsExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", gfid); + + Assert.Throws(() => + XfsExtendedAttributes.Set(image, "brick/data.txt", "trusted.too-large", new byte[128])); + + Assert.That(XfsExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(gfid)); + } +} From 3db82097d5f6dba5a6bf6db892dee352d99c9c8c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:25:20 +0200 Subject: [PATCH 135/165] Track backing paths for Gluster xattrs --- FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs b/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs index acbf0813d..936f723d6 100644 --- a/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs +++ b/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs @@ -30,4 +30,9 @@ public sealed class GlusterFsEntry { /// Lazily materializes an entry delegated to the backing filesystem reader. /// internal Func? DataFactory { get; init; } + + /// + /// Native path inside the backing filesystem, used for xattr lookup. + /// + internal string? BackingPath { get; init; } } From fd67adcf5387722ee4b1969c47834334969217c0 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:25:54 +0200 Subject: [PATCH 136/165] Expose Gluster brick xattrs through backing readers --- .../FileSystem.GlusterFs/GlusterFsReader.cs | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs b/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs index 6b25fdc42..7ae715059 100644 --- a/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs +++ b/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs @@ -25,8 +25,10 @@ namespace FileSystem.GlusterFs; /// present, the filesystem root is used as a conservative explicit-input /// fallback. /// -/// The backing readers currently do not interpret or mutate Gluster xattrs. -/// Those bytes remain untouched because this type is read-only. +/// Gluster xattrs are readable through . +/// The backing accessors also support conservative inline/short-form mutation, +/// but this brick reader remains read-only until all xattr storage forms needed +/// by a maintenance operation can be preserved without rebuilding metadata. /// public sealed class GlusterFsReader : IDisposable { @@ -132,6 +134,7 @@ private void ParseXfs() { Name = BrickPath(relativePath), Size = entry.Size, IsDirectory = entry.IsDirectory, + BackingPath = NormalizePath(entry.Name), DataFactory = entry.IsDirectory ? null : () => _xfsReader!.Extract(captured), }); } @@ -152,6 +155,7 @@ private void ParseExt() { Name = BrickPath(relativePath), Size = entry.Size, IsDirectory = entry.IsDirectory, + BackingPath = NormalizePath(entry.Name), DataFactory = entry.IsDirectory ? null : () => _extReader!.Extract(captured), }); } @@ -220,10 +224,12 @@ private byte[] BuildMetadata(int backingEntryCount) { builder.Append(CultureInfo.InvariantCulture, $"gluster_index_detected={this.HasGlusterIndex.ToString().ToLowerInvariant()}\n"); builder.Append("view=physical single-brick namespace\n"); builder.Append("gluster_internal_directory=.glusterfs (hidden from normal listing)\n"); - builder.Append("xattrs=preserved in image but not interpreted\n"); + builder.Append("xattrs=readable through native backing-filesystem accessors\n"); + builder.Append("xattr_mutation=native inline/short-form subset only; descriptor remains read-only\n"); builder.Append("cluster_namespace_reconstruction=false\n"); + builder.Append("cluster_operations=rebalance,fix-layout,remove-brick are outside the single-image abstraction\n"); builder.Append("mutation=false\n"); - builder.Append("mutation_blocker=Gluster object identity and DHT/AFR/EC state live in trusted.gfid/trusted.glusterfs.* xattrs; current backing writers do not preserve those attributes during rebuilds.\n"); + builder.Append("mutation_blocker=maintenance must preserve every trusted.gfid/trusted.glusterfs.* storage form; external/leaf/btree xattr mutation is not complete yet.\n"); return Encoding.UTF8.GetBytes(builder.ToString()); } @@ -258,6 +264,22 @@ public byte[] Extract(GlusterFsEntry entry) { return entry.DataFactory?.Invoke() ?? entry.Data; } + /// + /// Reads the native extended attributes belonging to a surfaced brick entry. + /// + public IReadOnlyDictionary ReadExtendedAttributes(GlusterFsEntry entry) { + ArgumentNullException.ThrowIfNull(entry); + if (entry.BackingPath is null) + return new Dictionary(StringComparer.Ordinal); + + _image.Position = 0; + return this.BackingFileSystem switch { + "xfs" => XfsExtendedAttributes.Read(_image, entry.BackingPath), + "ext" => ExtExtendedAttributes.Read(_image, entry.BackingPath), + _ => throw new NotSupportedException($"GlusterFS: xattrs are not supported for backing filesystem '{this.BackingFileSystem}'."), + }; + } + /// /// Releases backing filesystem readers and any spool created for a non-seekable input. /// From 3031bc5a1354eae066184c2f6f63880a487f3f81 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:25:56 +0200 Subject: [PATCH 137/165] Guard NRG rebuild verbs by track profile --- .../FileFormat.Nrg/NrgStructureInspector.cs | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 FileFormats/FileFormat.Nrg/NrgStructureInspector.cs diff --git a/FileFormats/FileFormat.Nrg/NrgStructureInspector.cs b/FileFormats/FileFormat.Nrg/NrgStructureInspector.cs new file mode 100644 index 000000000..127403774 --- /dev/null +++ b/FileFormats/FileFormat.Nrg/NrgStructureInspector.cs @@ -0,0 +1,131 @@ +using System.Buffers.Binary; + +namespace FileFormat.Nrg; + +/// Minimal descriptor-chain inspection shared by mutation/maintenance profile guards. +internal static class NrgStructureInspector { + internal readonly record struct Profile(int TrackCount, byte? SingleTrackMode) { + internal bool IsSingleDataTrack => this.TrackCount == 1 && this.SingleTrackMode is + 0x00 or 0x02 or 0x03 or 0x05 or 0x06 or 0x0F or 0x11; + } + + internal static Profile Inspect(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead || !stream.CanSeek) + throw new ArgumentException("NRG inspection requires a readable, seekable stream.", nameof(stream)); + + var saved = stream.Position; + try { + if (!TryGetDescriptorBounds(stream, out var descriptorOffset, out var footerOffset)) + return default; + + var trackCount = 0; + byte? singleMode = null; + var position = descriptorOffset; + Span header = stackalloc byte[8]; + while (position <= footerOffset - header.Length) { + stream.Position = position; + if (!ReadExactly(stream, header)) + break; + + var payloadLength = BinaryPrimitives.ReadUInt32BigEndian(header[4..]); + var payloadStart = position + header.Length; + if (payloadStart > footerOffset || payloadLength > (ulong)(footerOffset - payloadStart)) + break; + + if (header[..4].SequenceEqual("DAOX"u8)) + CountDao(stream, payloadStart, payloadLength, 42, ref trackCount, ref singleMode); + else if (header[..4].SequenceEqual("DAOI"u8)) + CountDao(stream, payloadStart, payloadLength, 30, ref trackCount, ref singleMode); + else if (header[..4].SequenceEqual("ETN2"u8)) + CountEtn(stream, payloadStart, payloadLength, 32, modeOffset: 19, ref trackCount, ref singleMode); + else if (header[..4].SequenceEqual("ETNF"u8)) + CountEtn(stream, payloadStart, payloadLength, 20, modeOffset: 11, ref trackCount, ref singleMode); + + position = payloadStart + payloadLength; + if (header[..4].SequenceEqual("END!"u8)) + break; + } + + return new(trackCount, trackCount == 1 ? singleMode : null); + } finally { + stream.Position = saved; + } + } + + private static void CountDao(Stream stream, long payloadStart, uint payloadLength, int recordSize, + ref int trackCount, ref byte? singleMode) { + const int headerSize = 22; + if (payloadLength < headerSize) + return; + var records = checked((int)((payloadLength - headerSize) / recordSize)); + for (var index = 0; index < records; ++index) { + stream.Position = payloadStart + headerSize + (long)index * recordSize + 14; + var mode = stream.ReadByte(); + if (mode < 0) + return; + AddMode((byte)mode, ref trackCount, ref singleMode); + } + } + + private static void CountEtn(Stream stream, long payloadStart, uint payloadLength, int recordSize, int modeOffset, + ref int trackCount, ref byte? singleMode) { + var records = checked((int)(payloadLength / recordSize)); + for (var index = 0; index < records; ++index) { + stream.Position = payloadStart + (long)index * recordSize + modeOffset; + var mode = stream.ReadByte(); + if (mode < 0) + return; + AddMode((byte)mode, ref trackCount, ref singleMode); + } + } + + private static void AddMode(byte mode, ref int trackCount, ref byte? singleMode) { + ++trackCount; + singleMode = trackCount == 1 ? mode : null; + } + + private static bool TryGetDescriptorBounds(Stream stream, out long descriptorOffset, out long footerOffset) { + descriptorOffset = 0; + footerOffset = 0; + + if (stream.Length >= 12) { + stream.Position = stream.Length - 12; + Span footer = stackalloc byte[12]; + if (ReadExactly(stream, footer) && footer[..4].SequenceEqual("NER5"u8)) { + var offset = BinaryPrimitives.ReadUInt64BigEndian(footer[4..]); + footerOffset = stream.Length - 12; + if (offset <= (ulong)footerOffset) { + descriptorOffset = checked((long)offset); + return true; + } + } + } + + if (stream.Length >= 8) { + stream.Position = stream.Length - 8; + Span footer = stackalloc byte[8]; + if (ReadExactly(stream, footer) && footer[..4].SequenceEqual("NERO"u8)) { + var offset = BinaryPrimitives.ReadUInt32BigEndian(footer[4..]); + footerOffset = stream.Length - 8; + if (offset <= footerOffset) { + descriptorOffset = offset; + return true; + } + } + } + + return false; + } + + private static bool ReadExactly(Stream stream, Span buffer) { + var offset = 0; + while (offset < buffer.Length) { + var read = stream.Read(buffer[offset..]); + if (read == 0) + return false; + offset += read; + } + return true; + } +} From 4272f0dd5ed7ab60999291af47d7dddb231df12c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:26:31 +0200 Subject: [PATCH 138/165] Verify Gluster brick xattr visibility --- .../GlusterFs/GlusterFsDetectionTests.cs | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs b/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs index ef0ff52e2..cdd2ef183 100644 --- a/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs +++ b/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs @@ -1,28 +1,45 @@ using System.Text; using Compression.Registry; +using FileSystem.Ext; using FileSystem.GlusterFs; +using FileSystem.Xfs; namespace Compression.Tests.GlusterFs; [TestFixture] public class GlusterFsDetectionTests { + private static readonly byte[] Gfid = [ + 0x10, 0x32, 0x54, 0x76, 0x98, 0xBA, 0xDC, 0xFE, + 0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, + ]; + private static byte[] BuildXfsBrick() { - var writer = new FileSystem.Xfs.XfsWriter(); + var writer = new XfsWriter(); writer.AddFile("host/brick/data/hello.txt", "hello from xfs brick"u8.ToArray()); writer.AddFile("host/brick/.glusterfs/ab/internal-gfid", "backend index"u8.ToArray()); writer.AddFile("outside.txt", "not part of the brick"u8.ToArray()); using var stream = new MemoryStream(); writer.WriteTo(stream); + XfsExtendedAttributes.Set(stream, "host/brick/data/hello.txt", "trusted.gfid", Gfid); return stream.ToArray(); } private static byte[] BuildExtBrick() { - var writer = new FileSystem.Ext.ExtWriter(); + var writer = new ExtWriter(); writer.AddFile("srv/brick/data/hello.txt", "hello from ext brick"u8.ToArray()); writer.AddFile("srv/brick/.glusterfs/cd/internal-gfid", "backend index"u8.ToArray()); writer.AddFile("outside.txt", "not part of the brick"u8.ToArray()); - return writer.Build(); + var bytes = writer.Build( + blockSize: 1024, + totalBlocks: 4096, + version: ExtWriter.ExtVersion.Ext4, + journal: false, + volumeLabel: "gluster", + inodeSize: 256); + using var stream = new MemoryStream(bytes, writable: true); + ExtExtendedAttributes.Set(stream, "srv/brick/data/hello.txt", "trusted.gfid", Gfid); + return stream.ToArray(); } [Test, Category("HappyPath")] @@ -70,11 +87,21 @@ public void ExtBackingStore_InfersBrickRootAndExtractsBrickFile() { Assert.That(reader.BrickRoot, Is.EqualTo("srv/brick")); Assert.That(reader.HasGlusterIndex, Is.True); Assert.That(Encoding.UTF8.GetString(reader.Extract(entry)), Is.EqualTo("hello from ext brick")); + Assert.That(reader.ReadExtendedAttributes(entry)["trusted.gfid"], Is.EqualTo(Gfid)); Assert.That(reader.Entries.Any(candidate => candidate.Name.Contains(".glusterfs", StringComparison.Ordinal)), Is.False); Assert.That(reader.Entries.Any(candidate => candidate.Name == "brick/outside.txt"), Is.False); }); } + [Test, Category("HappyPath")] + public void XfsBackingStore_SurfacesNativeGlusterXattrs() { + using var stream = new MemoryStream(BuildXfsBrick()); + using var reader = new GlusterFsReader(stream); + var entry = reader.Entries.Single(candidate => candidate.Name == "brick/data/hello.txt"); + + Assert.That(reader.ReadExtendedAttributes(entry)["trusted.gfid"], Is.EqualTo(Gfid)); + } + [Test, Category("HappyPath")] public void XfsBackingStore_ExtractsThroughDescriptor() { var descriptor = new GlusterFsFormatDescriptor(); @@ -101,7 +128,7 @@ public void WorkbenchProbeMagic_IsNoLongerAcceptedAsGlusterFs() { } [Test, Category("Regression")] - public void Descriptor_DoesNotAdvertiseMutationsThatWouldDiscardGlusterXattrs() { + public void Descriptor_DoesNotAdvertiseMutationsUntilAllXattrStorageFormsAreSafe() { var descriptor = new GlusterFsFormatDescriptor(); Assert.Multiple(() => { @@ -116,7 +143,7 @@ public void Descriptor_DoesNotAdvertiseMutationsThatWouldDiscardGlusterXattrs() } [Test, Category("HappyPath")] - public void Metadata_DescribesPhysicalViewBrickRootAndMutationBlocker() { + public void Metadata_DescribesXattrProgressAndClusterOperationBoundary() { using var stream = new MemoryStream(BuildXfsBrick()); using var reader = new GlusterFsReader(stream); var metadata = reader.Entries.Single(entry => entry.Name == "metadata.ini"); @@ -127,10 +154,11 @@ public void Metadata_DescribesPhysicalViewBrickRootAndMutationBlocker() { Assert.That(text, Does.Contain("backing_fs=xfs")); Assert.That(text, Does.Contain("brick_root=host/brick")); Assert.That(text, Does.Contain("gluster_index_detected=true")); - Assert.That(text, Does.Contain("view=physical single-brick namespace")); - Assert.That(text, Does.Contain("xattrs=preserved in image but not interpreted")); + Assert.That(text, Does.Contain("xattrs=readable through native backing-filesystem accessors")); + Assert.That(text, Does.Contain("xattr_mutation=native inline/short-form subset only")); Assert.That(text, Does.Contain("cluster_namespace_reconstruction=false")); - Assert.That(text, Does.Contain("trusted.gfid/trusted.glusterfs.*")); + Assert.That(text, Does.Contain("cluster_operations=rebalance,fix-layout,remove-brick are outside the single-image abstraction")); + Assert.That(text, Does.Contain("external/leaf/btree xattr mutation is not complete yet")); }); } } From edf2fabbcc3c2df9a607596a8a822785338db36c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:26:31 +0200 Subject: [PATCH 139/165] Route NRG creation through DAO writer --- .../FileFormat.Nrg/NrgFormatDescriptor.cs | 136 ++++++++++++------ 1 file changed, 94 insertions(+), 42 deletions(-) diff --git a/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs b/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs index e45405df4..e904a94fb 100644 --- a/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs +++ b/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs @@ -1,5 +1,4 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; using Compression.Registry; using static Compression.Registry.FormatHelpers; @@ -12,15 +11,13 @@ namespace FileFormat.Nrg; /// References: /// /// https://cdemu.sourceforge.io — CDEmu / libMirage NRG parser, used as a behavioural oracle for the reverse-engineered chunk layout -/// https://en.wikipedia.org/wiki/NRG_(file_format) — format overview +/// https://problemkaputt.de/psx-spx.htm — independently documented NRG CUEX/DAOX/ETN structures /// No official public Nero specification is available; the format is proprietary and reverse-engineered /// /// public sealed class NrgFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable, IArchiveDefragmentable, IArchiveShrinkable { - private const uint CdRomMediumType = 0x00000400; - /// Gets the id. public string Id => "Nrg"; @@ -52,7 +49,10 @@ public sealed class NrgFormatDescriptor : IFormatDescriptor, IArchiveFormatOpera public IReadOnlyList MagicSignatures => []; /// Gets the methods. - public IReadOnlyList Methods => [new("iso9660", "ISO 9660")]; + public IReadOnlyList Methods => [ + new("iso9660", "ISO 9660"), + new("cdda", "CD-DA / mixed-mode tracks"), + ]; /// Gets the tar compression format id. public string? TarCompressionFormatId => null; @@ -62,16 +62,16 @@ public sealed class NrgFormatDescriptor : IFormatDescriptor, IArchiveFormatOpera /// Gets the description. public string Description => - "Nero Burning ROM disc image (NRG v1/v2 reader; interoperable v2 TAO ISO writer; R/W and maintenance through verified rebuild)"; + "Nero Burning ROM disc image (NRG v1/v2 reader; v2 DAO multi-session/multi-track/audio writer; named ISO edits use verified rebuild)"; - /// Lists the entries in the supplied container. + /// Lists the ISO 9660 entries from the first readable data track in the supplied container. public List List(Stream stream, string? password) { using var reader = new NrgReader(stream, leaveOpen: true); return reader.Entries.Select((entry, index) => new ArchiveEntryInfo(index, entry.FullPath, entry.Size, entry.Size, "iso9660", entry.IsDirectory, false, null)).ToList(); } - /// Decodes the supplied input. + /// Extracts ISO 9660 entries from the first readable data track. public void Extract(Stream stream, string outputDir, string? password, string[]? files) { using var reader = new NrgReader(stream, leaveOpen: true); foreach (var entry in reader.Entries) { @@ -84,11 +84,9 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? } /// - /// Creates a single-session, single-track NRG v2 image containing a cooked - /// 2,048-byte/sector ISO 9660 track. The trailer is a real NRG descriptor - /// chain rather than merely a NER5 signature: ETN2 describes the TAO track, - /// SINF records the session track count, MTYP marks CD-ROM media, and END! - /// terminates the chunk list. + /// Creates the generic archive API profile: one DAO session containing one cooked + /// Mode-1 ISO 9660 track. Call + /// directly for multi-session, mixed data/audio, pregap, MCN, ISRC, CD-TEXT or raw-sector authoring. /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { ArgumentNullException.ThrowIfNull(output); @@ -98,42 +96,96 @@ public void Create(Stream output, IReadOnlyList inputs, Format foreach (var input in inputs.Where(static input => !input.IsDirectory)) iso.AddFile(input.ArchiveName.Replace('\\', '/'), input.ReadContent()); - var image = iso.Build(); - output.Write(image); - - var trailerOffset = checked((ulong)output.Position); + NrgWriter.Write(output, new NrgDiscDefinition([ + new NrgSessionDefinition([ + new NrgTrackDefinition(NrgTrackMode.Mode1, iso.Build()), + ]), + ])); + } - Span etn2 = stackalloc byte[32]; - BinaryPrimitives.WriteUInt64BigEndian(etn2, 0); // track starts at file offset 0 - BinaryPrimitives.WriteUInt64BigEndian(etn2[8..], checked((ulong)image.LongLength)); - etn2[19] = 0x00; // Mode 1, cooked 2,048-byte user-data sectors - BinaryPrimitives.WriteUInt32BigEndian(etn2[20..], 0); // first logical sector - WriteChunk(output, "ETN2"u8, etn2); + /// + /// Adds/replaces named ISO entries through a verified rebuild. Multi-track or audio NRGs are + /// deliberately refused here because flattening them to the generic single-ISO create profile + /// would destroy disc structure. + /// + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(inputs); + EnsureNamedFileMutationProfile(archive); + RebuildVerb.EditViaRebuild(archive, this, this, tmpDir => { + foreach (var input in inputs) { + if (input.IsDirectory || string.IsNullOrEmpty(input.ArchiveName)) + continue; + var destination = Path.Combine(tmpDir, input.ArchiveName.Replace('/', Path.DirectorySeparatorChar)); + var parent = Path.GetDirectoryName(destination); + if (!string.IsNullOrEmpty(parent)) + Directory.CreateDirectory(parent); + File.WriteAllBytes(destination, input.ReadContent()); + } + }); + } - Span sinf = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32BigEndian(sinf, 1); // one track in this session - WriteChunk(output, "SINF"u8, sinf); + /// Removes named ISO entries through the same profile-gated verified rebuild. + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(entryNames); + EnsureNamedFileMutationProfile(archive); + var skip = new HashSet(entryNames, StringComparer.OrdinalIgnoreCase); + RebuildVerb.EditViaRebuild(archive, this, this, tmpDir => { + foreach (var file in Directory.GetFiles(tmpDir, "*", SearchOption.AllDirectories)) { + var relative = Path.GetRelativePath(tmpDir, file).Replace('\\', '/'); + if (skip.Contains(relative) || skip.Contains(Path.GetFileName(relative))) + File.Delete(file); + } + }); + } - Span mtyp = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32BigEndian(mtyp, CdRomMediumType); - WriteChunk(output, "MTYP"u8, mtyp); + /// Purges the single-data-track R/W profile to a valid empty NRG. + public void Purge(Stream archive) { + EnsureNamedFileMutationProfile(archive); + RebuildVerb.PurgeViaModifier(archive, this, this); + } - WriteChunk(output, "END!"u8, ReadOnlySpan.Empty); + /// Rebuild-defragments the single-data-track R/W profile. + public void Defragment(Stream archive) { + EnsureNamedFileMutationProfile(archive); + RebuildVerb.RebuildInPlace(archive, this, this); + } - Span footer = stackalloc byte[12]; - "NER5"u8.CopyTo(footer); - BinaryPrimitives.WriteUInt64BigEndian(footer[4..], trailerOffset); - output.Write(footer); + /// Progress-reporting rebuild defrag for the single-data-track R/W profile. + public void Defragment(Stream archive, DefragOptions options) { + ArgumentNullException.ThrowIfNull(options); + if (options.Mode != DefragMode.ConsolidateAtStart) + throw new NotSupportedException($"NRG rebuild defrag supports only {DefragMode.ConsolidateAtStart}."); + EnsureNamedFileMutationProfile(archive); + RebuildVerb.RebuildInPlace(archive, this, this, + onProgress: options.OnProgress, + cancellationToken: options.CancellationToken); } - private static void WriteChunk(Stream output, ReadOnlySpan id, ReadOnlySpan payload) { - if (id.Length != 4) - throw new ArgumentException("NRG chunk ids are exactly four bytes.", nameof(id)); + /// + /// Tight-packs the single-data-track profile. Multi-track/audio images are copied through + /// unchanged rather than being flattened into one ISO track. + /// + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + var profile = NrgStructureInspector.Inspect(input); + if (!profile.IsSingleDataTrack) { + input.Position = 0; + output.Position = 0; + output.SetLength(0); + input.CopyTo(output); + return; + } + + ((IArchiveShrinkable)this).ShrinkDefault(input, output); + } - Span header = stackalloc byte[8]; - id.CopyTo(header); - BinaryPrimitives.WriteUInt32BigEndian(header[4..], checked((uint)payload.Length)); - output.Write(header); - output.Write(payload); + private static void EnsureNamedFileMutationProfile(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + var profile = NrgStructureInspector.Inspect(archive); + if (!profile.IsSingleDataTrack) + throw new NotSupportedException( + "Named-file mutation/defrag/purge is supported only for a single data-track NRG. " + + "Multi-session, mixed-mode and audio images are authorable but are not flattened during file-level maintenance."); } } From 56f3996fe66c8532139fd4fe9f6621e3135e053c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:27:05 +0200 Subject: [PATCH 140/165] docs(onefs): record simulator oracle path --- FileSystems/FileSystem.OneFs/FORMAT-NOTES.md | 40 ++++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md b/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md index 28ff5d161..35beb1904 100644 --- a/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md +++ b/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md @@ -109,6 +109,34 @@ missing proprietary header definitions. It is therefore treated only as a source of behavioral/symbol-name clues; no code, layout or expressive structure from it is reused here. +## Genuine-media oracle path + +Dell publicly distributes a OneFS Simulator specifically for testing. Dell's own +community linked the 9.5.0.0 simulator ZIP, and the simulator installation guide +states that the ZIP contains an OVA. Independent inspection of that OVA reports a +small sparse appliance with 22 VMDKs: one 16 GiB system disk, one 512 MiB device, +and twenty 5 GiB SCSI data disks. Those data VMDKs are the best available oracle +for the next raw-format step because they are genuine Dell-authored OneFS media, +not synthetic bytes invented by this repository. + +The intended clean-room oracle procedure is: + +1. obtain the simulator from Dell's own download channel; +2. extract the OVA/VMDKs without redistributing them; +3. identify which 5 GiB VMDKs are initialized `/ifs` data drives; +4. reconstruct each sparse VMDK as logical raw sectors using this repository's + existing VMDK reader; +5. compare the same 8 KiB block positions across many drives and fresh simulator + clusters to find the repeated fixed-address superblock candidates; +6. change one controlled cluster property at a time, then diff those candidates; +7. validate any inferred magic/version/root-address fields against live `isi get`, + cluster upgrade behavior and more than one simulator version before adding a + detector or parser. + +This run could verify the public simulator packaging and disk inventory, but the +execution environment could not resolve `dl.dell.com`, so no simulator bytes were +used and no claim about the missing raw offsets or magic is made here. + ## What remains unknown at byte level No authoritative publication or independently licensed implementation found in @@ -176,6 +204,10 @@ Primary / official material: - Dell KB, **How to obtain a LIN from an NFS file handle** (little-endian exported LIN encoding): https://www.dell.com/support/kbdoc/en-us/000019498/isilon-onefs-how-to-obtain-a-lin-from-an-nfs-file-handle +- Dell OneFS Simulator Installation Guide: + https://www.dell.com/support/manuals/en-us/isilon-onefs/ifs_pub_onefs_simulator_guide/installing-onefs-simulator +- Dell community, **Upgrading OneFS Simulator** (Dell-hosted 9.5 simulator link): + https://www.dell.com/community/en/conversations/isilon/upgrading-onefs-simulator/647f8bdcf4ccf8a8dec07295 Implementation-independent patent evidence: @@ -193,6 +225,8 @@ Implementation-independent patent evidence: Dell documentation and patents are used only to determine factual architecture, public behavior, constants/geometry and acceptance boundaries. Patent pseudocode is not copied or translated. The unlicensed `cp-migrate` repository is not a code -source. No external implementation code, comments, naming structure or control -flow is reproduced. The implementation remains original managed C# under this -repository's license, with no new dependency. +source. Dell simulator binaries are not committed or redistributed; when +available they are suitable only as a behavioral/raw-media oracle. No external +implementation code, comments, naming structure or control flow is reproduced. +The implementation remains original managed C# under this repository's license, +with no new dependency. From e248684ef555a28e31f006fe9469d4933da93046 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:27:29 +0200 Subject: [PATCH 141/165] Document Gluster xattr and cluster-operation boundaries --- .../GlusterFsFormatDescriptor.cs | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs b/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs index f6b1487dd..29949dd9a 100644 --- a/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs +++ b/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs @@ -20,19 +20,26 @@ namespace FileSystem.GlusterFs; /// which to distinguish a brick image automatically. The former workbench-only /// 0xCAFE5BAB probe convention is deliberately not recognised as GlusterFS. /// -/// Mutating maintenance operations are intentionally not advertised. A -/// Gluster brick relies on trusted.gfid and trusted.glusterfs.* -/// xattrs for identity, DHT layout, replication/heal and related state. The -/// current backing-filesystem rebuild paths do not preserve those attributes, -/// so create/modify/defrag/shrink/layout/wipe/purge would risk silent brick -/// corruption. +/// The native XFS/ext layers can now read Gluster xattrs and conservatively +/// mutate the common short-form/in-inode storage cases. This descriptor remains +/// read-only until every xattr storage form that a maintenance operation can +/// encounter (including ext external blocks/EA inodes and XFS leaf/btree/remote +/// attributes) is preserved. Advertising a rebuild-based mutation before then +/// could silently discard trusted.gfid or trusted.glusterfs.* state. +/// +/// Gluster volume operations such as rebalance, fix-layout, and remove-brick +/// are explicitly outside this single-image abstraction. They coordinate multiple +/// bricks and belong to a live cluster/volume control plane, not an offline brick +/// image editor. /// /// References: /// /// https://docs.gluster.org/en/latest/Administrator-Guide/Setting-Up-Volumes/ — a volume is a logical collection of export-directory bricks /// https://docs.gluster.org/en/latest/Administrator-Guide/GlusterFS-Introduction/ — backing filesystems must support extended attributes -/// https://docs.gluster.org/en/latest/Administrator-Guide/Managing-Volumes/ — shrink/rebalance/fix-layout are cluster operations -/// https://github.com/gluster/glusterfs — canonical implementation, dual GPLv2/LGPLv3+ +/// https://docs.gluster.org/en/latest/Administrator-Guide/Managing-Volumes/ — remove-brick/rebalance/fix-layout are volume operations +/// https://docs.kernel.org/filesystems/ext4/attributes.html — ext4 xattr on-disk layout +/// https://www.kernel.org/pub/linux/utils/fs/xfs/docs/xfs_filesystem_structure.pdf — XFS short-form attribute layout +/// https://github.com/gluster/glusterfs — canonical GlusterFS implementation, dual GPLv2/LGPLv3+ /// /// public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { @@ -89,9 +96,10 @@ public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveForma public string Description => "GlusterFS single-brick R/O backing-store view via XFS/ext delegation. GlusterFS has no " + "standalone image format; the supplied .gluster image is treated as one brick's backing " + - "filesystem. The .glusterfs GFID index is hidden from normal listings. trusted.gfid and " + - "trusted.glusterfs.* xattrs remain preserved in the raw image but are not interpreted, so " + - "cluster namespace reconstruction and all mutations remain disabled."; + "filesystem. The .glusterfs GFID index is hidden from normal listings. Native backing " + + "accessors can read trusted.gfid/trusted.glusterfs.* and mutate inline/short-form xattrs, " + + "but archive mutations stay disabled until external/leaf/btree xattr forms are safe. " + + "Cluster rebalance, fix-layout, and remove-brick are intentionally out of scope."; /// /// Lists the entries in the supplied brick backing-store image. From 4c64c4f476ad3995c8360a9c183820390e14d9db Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:28:25 +0200 Subject: [PATCH 142/165] fix(onefs): stop claiming opaque-image integrity testing --- .../FileSystem.OneFs/OneFsFormatDescriptor.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs b/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs index 1ea9ea4a2..bd2fbf68f 100644 --- a/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs +++ b/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs @@ -65,9 +65,13 @@ public sealed class OneFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOpe /// Gets the format category. public FormatCategory Category => FormatCategory.Archive; - /// Gets the conservative read-only capabilities. + /// + /// Gets the conservative inspection capabilities. Integrity testing is not + /// advertised: without a verified superblock/tree parser, successfully copying + /// an opaque image does not prove that the filesystem is structurally sound. + /// public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest; + FormatCapabilities.CanList | FormatCapabilities.CanExtract; /// Gets the conventional extension for explicitly supplied raw media. public string DefaultExtension => ".onefs"; @@ -100,8 +104,8 @@ public sealed class OneFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOpe "through mirrored node/drive/block addresses and distributed protection groups. Fixed-address superblocks point " + "toward the LIN master, but Dell does not publish the byte-level superblock/tree/allocation serialization or an " + "authoritative offset-zero media magic. The historic 'OneFS'/'ONEF' signature claim was therefore removed. " + - "R/W, purge, wipe, defrag, shrink and layout rebuild remain blocked until allocation/protection/journal updates can " + - "be proven against a real cluster or independent checker."; + "Integrity testing, R/W, purge, wipe, defrag, shrink and layout rebuild remain blocked until the raw structures " + + "and allocation/protection/journal updates can be proven against genuine media and an independent checker."; /// Lists the two conservative inspection entries without reading the image payload. public List List(Stream stream, string? password) { From b01875373488ce7800c6b858b3a6530cc6c0949c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:28:48 +0200 Subject: [PATCH 143/165] Test NRG multi-session mixed-mode authoring --- Compression.Tests/Nrg/NrgAuthoringTests.cs | 266 +++++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 Compression.Tests/Nrg/NrgAuthoringTests.cs diff --git a/Compression.Tests/Nrg/NrgAuthoringTests.cs b/Compression.Tests/Nrg/NrgAuthoringTests.cs new file mode 100644 index 000000000..4ce750dd9 --- /dev/null +++ b/Compression.Tests/Nrg/NrgAuthoringTests.cs @@ -0,0 +1,266 @@ +using System.Buffers.Binary; +using Compression.Registry; +using FileFormat.Nrg; + +namespace Compression.Tests.Nrg; + +[TestFixture] +public class NrgAuthoringTests { + private const int CookedSectorSize = 2048; + private const int AudioSectorSize = 2352; + + [Test, Category("RoundTrip")] + public void Write_MixedModeDao_PreservesAudioAndFindsDataTrack() { + var audio = Pattern(AudioSectorSize * 3, 17); + var iso = BuildIso("SECOND.BIN", "second-track"u8.ToArray()); + var disc = new NrgDiscDefinition([ + new NrgSessionDefinition([ + new NrgTrackDefinition(NrgTrackMode.Audio, audio) { + PregapSectors = 150, + Isrc = "USABC2600001", + }, + new NrgTrackDefinition(NrgTrackMode.Mode1, iso), + ]) { Mcn = "1234567890123" }, + ]); + + var image = Write(disc); + var chunks = ReadChunks(image); + Assert.That(chunks.Select(static chunk => chunk.Id), Is.EqualTo(new[] { "CUEX", "DAOX", "SINF", "MTYP", "END!" })); + + var cuex = chunks.Single(static chunk => chunk.Id == "CUEX").Payload; + Assert.That(cuex.Length, Is.EqualTo(48), "lead-in + two indices per track + lead-out"); + AssertCue(cuex, 0, adrCtl: 0x01, track: 0x00, index: 0x00, lba: -150); + AssertCue(cuex, 1, adrCtl: 0x01, track: 0x01, index: 0x00, lba: 0); + AssertCue(cuex, 2, adrCtl: 0x01, track: 0x01, index: 0x01, lba: 150); + AssertCue(cuex, 3, adrCtl: 0x41, track: 0x02, index: 0x00, lba: 153); + AssertCue(cuex, 4, adrCtl: 0x41, track: 0x02, index: 0x01, lba: 153); + AssertCue(cuex, 5, adrCtl: 0x41, track: 0xAA, index: 0x01, lba: 153 + iso.Length / CookedSectorSize); + + var daox = chunks.Single(static chunk => chunk.Id == "DAOX").Payload; + Assert.That(daox.Length, Is.EqualTo(22 + 2 * 42)); + Assert.Multiple(() => { + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(daox), Is.EqualTo((uint)daox.Length)); + Assert.That(System.Text.Encoding.ASCII.GetString(daox, 4, 13), Is.EqualTo("1234567890123")); + Assert.That(daox[20], Is.EqualTo(1)); + Assert.That(daox[21], Is.EqualTo(2)); + }); + + var audioRecord = daox.AsSpan(22, 42); + var dataRecord = daox.AsSpan(64, 42); + var audioPregap = checked((long)BinaryPrimitives.ReadUInt64BigEndian(audioRecord[18..])); + var audioStart = checked((long)BinaryPrimitives.ReadUInt64BigEndian(audioRecord[26..])); + var audioEnd = checked((long)BinaryPrimitives.ReadUInt64BigEndian(audioRecord[34..])); + var dataStart = checked((long)BinaryPrimitives.ReadUInt64BigEndian(dataRecord[26..])); + var dataEnd = checked((long)BinaryPrimitives.ReadUInt64BigEndian(dataRecord[34..])); + + Assert.Multiple(() => { + Assert.That(BinaryPrimitives.ReadUInt16BigEndian(audioRecord[12..]), Is.EqualTo(AudioSectorSize)); + Assert.That(audioRecord[14], Is.EqualTo((byte)NrgTrackMode.Audio)); + Assert.That(System.Text.Encoding.ASCII.GetString(audioRecord[..12]), Is.EqualTo("USABC2600001")); + Assert.That(audioPregap, Is.Zero); + Assert.That(audioStart - audioPregap, Is.EqualTo(150L * AudioSectorSize)); + Assert.That(audioEnd - audioStart, Is.EqualTo(audio.Length)); + Assert.That(BinaryPrimitives.ReadUInt16BigEndian(dataRecord[12..]), Is.EqualTo(CookedSectorSize)); + Assert.That(dataRecord[14], Is.EqualTo((byte)NrgTrackMode.Mode1)); + Assert.That(dataStart, Is.EqualTo(audioEnd)); + Assert.That(dataEnd - dataStart, Is.EqualTo(iso.Length)); + }); + + Assert.That(image.AsSpan(checked((int)audioStart), audio.Length).ToArray(), Is.EqualTo(audio)); + Assert.That(image.AsSpan(checked((int)dataStart + 16 * CookedSectorSize), 6).ToArray(), + Is.EqualTo(new byte[] { 1, (byte)'C', (byte)'D', (byte)'0', (byte)'0', (byte)'1' })); + + using var stream = new MemoryStream(image, writable: false); + using var reader = new NrgReader(stream); + var entry = reader.Entries.Single(e => !e.IsDirectory && e.Name.Equals("SECOND.BIN", StringComparison.OrdinalIgnoreCase)); + Assert.That(reader.Extract(entry), Is.EqualTo("second-track"u8.ToArray())); + } + + [Test, Category("RoundTrip")] + public void Write_MultipleSessions_UsesCdLeadOutAndLeadInSpacing() { + var firstIso = BuildIso("FIRST.BIN", "first"u8.ToArray()); + var secondIso = BuildIso("SECOND.BIN", "second"u8.ToArray()); + var image = Write(new NrgDiscDefinition([ + new NrgSessionDefinition([ + new NrgTrackDefinition(NrgTrackMode.Mode1, firstIso), + ]), + new NrgSessionDefinition([ + new NrgTrackDefinition(NrgTrackMode.Mode1, secondIso), + ]), + ])); + + var chunks = ReadChunks(image); + Assert.That(chunks.Select(static chunk => chunk.Id), Is.EqualTo(new[] { + "CUEX", "DAOX", "SINF", "CUEX", "DAOX", "SINF", "MTYP", "END!", + })); + + var cueSessions = chunks.Where(static chunk => chunk.Id == "CUEX").Select(static chunk => chunk.Payload).ToArray(); + var firstLeadOut = CueLba(cueSessions[0], 3); + var secondLeadIn = CueLba(cueSessions[1], 0); + var secondTrackIndex0 = CueLba(cueSessions[1], 1); + var secondTrackIndex1 = CueLba(cueSessions[1], 2); + + Assert.Multiple(() => { + Assert.That(secondLeadIn, Is.EqualTo(firstLeadOut + 6750), "first session has the long CD lead-out"); + Assert.That(secondTrackIndex0, Is.EqualTo(secondLeadIn + 4500), "next session starts after its lead-in"); + Assert.That(secondTrackIndex1, Is.EqualTo(secondTrackIndex0)); + Assert.That(cueSessions[0][9], Is.EqualTo(0x01), "first session uses track 1 BCD"); + Assert.That(cueSessions[1][9], Is.EqualTo(0x02), "track numbering remains global across sessions"); + }); + + var sinf = chunks.Where(static chunk => chunk.Id == "SINF").ToArray(); + Assert.That(sinf, Has.Length.EqualTo(2)); + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(sinf[0].Payload), Is.EqualTo(1)); + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(sinf[1].Payload), Is.EqualTo(1)); + } + + [Test, Category("RoundTrip")] + public void Write_CdText_WritesRaw18BytePacks() { + var cdText = Pattern(36, 91); + var image = Write(new NrgDiscDefinition([ + new NrgSessionDefinition([ + new NrgTrackDefinition(NrgTrackMode.Audio, Pattern(AudioSectorSize, 33)), + ]), + ]) { CdText = cdText }); + + var chunks = ReadChunks(image); + Assert.That(chunks.Select(static chunk => chunk.Id), Is.EqualTo(new[] { "CUEX", "DAOX", "SINF", "CDTX", "MTYP", "END!" })); + Assert.That(chunks.Single(static chunk => chunk.Id == "CDTX").Payload, Is.EqualTo(cdText)); + } + + [Test, Category("Boundary")] + public void Write_RejectsMisalignedAudioTrack() { + using var output = new MemoryStream(); + var disc = new NrgDiscDefinition([ + new NrgSessionDefinition([ + new NrgTrackDefinition(NrgTrackMode.Audio, new byte[AudioSectorSize + 1]), + ]), + ]); + + Assert.Throws(() => NrgWriter.Write(output, disc)); + } + + [Test, Category("Boundary")] + public void Write_RejectsMoreThan99Tracks() { + var tracks = Enumerable.Range(0, 100) + .Select(static _ => new NrgTrackDefinition(NrgTrackMode.Audio, new byte[AudioSectorSize])) + .ToArray(); + using var output = new MemoryStream(); + Assert.Throws(() => NrgWriter.Write(output, new NrgDiscDefinition([ + new NrgSessionDefinition(tracks), + ]))); + } + + [Test, Category("Boundary")] + public void Write_RejectsInvalidMcnIsrcAndCdText() { + using var output = new MemoryStream(); + Assert.Multiple(() => { + Assert.Throws(() => NrgWriter.Write(output, new NrgDiscDefinition([ + new NrgSessionDefinition([ + new NrgTrackDefinition(NrgTrackMode.Audio, new byte[AudioSectorSize]), + ]) { Mcn = "123" }, + ]))); + + Assert.Throws(() => NrgWriter.Write(output, new NrgDiscDefinition([ + new NrgSessionDefinition([ + new NrgTrackDefinition(NrgTrackMode.Audio, new byte[AudioSectorSize]) { Isrc = "short" }, + ]), + ]))); + + Assert.Throws(() => NrgWriter.Write(output, new NrgDiscDefinition([ + new NrgSessionDefinition([ + new NrgTrackDefinition(NrgTrackMode.Audio, new byte[AudioSectorSize]), + ]), + ]) { CdText = new byte[17] })); + }); + } + + [Test, Category("Regression")] + public void MultiTrack_FileMutationAndDefragRefuseWithoutChangingImage() { + var descriptor = new NrgFormatDescriptor(); + var source = Write(new NrgDiscDefinition([ + new NrgSessionDefinition([ + new NrgTrackDefinition(NrgTrackMode.Audio, Pattern(AudioSectorSize, 4)), + new NrgTrackDefinition(NrgTrackMode.Mode1, BuildIso("KEEP.BIN", "keep"u8.ToArray())), + ]), + ])); + + using var image = new MemoryStream(); + image.Write(source); + image.Position = 0; + + Assert.Throws(() => ((IArchiveModifiable)descriptor).Add(image, [ + ArchiveInputInfo.InMemory("NOPE.BIN", "nope"u8.ToArray()), + ])); + Assert.That(image.ToArray(), Is.EqualTo(source)); + + image.Position = 0; + Assert.Throws(() => ((IArchiveDefragmentable)descriptor).Defragment(image)); + Assert.That(image.ToArray(), Is.EqualTo(source)); + } + + [Test, Category("Regression")] + public void MultiTrack_ShrinkCopiesThroughByteIdentical() { + var descriptor = new NrgFormatDescriptor(); + var source = Write(new NrgDiscDefinition([ + new NrgSessionDefinition([ + new NrgTrackDefinition(NrgTrackMode.Audio, Pattern(AudioSectorSize * 2, 19)), + new NrgTrackDefinition(NrgTrackMode.Mode1, BuildIso("KEEP.BIN", "keep"u8.ToArray())), + ]), + ])); + + using var input = new MemoryStream(source, writable: false); + using var output = new MemoryStream(); + ((IArchiveShrinkable)descriptor).Shrink(input, output); + Assert.That(output.ToArray(), Is.EqualTo(source)); + } + + private static byte[] BuildIso(string name, byte[] data) { + var writer = new FileSystem.Iso.IsoWriter(); + writer.AddFile(name, data); + return writer.Build(); + } + + private static byte[] Write(NrgDiscDefinition disc) { + using var output = new MemoryStream(); + NrgWriter.Write(output, disc); + return output.ToArray(); + } + + private static byte[] Pattern(int length, int multiplier) + => Enumerable.Range(0, length).Select(i => unchecked((byte)(i * multiplier + 3))).ToArray(); + + private sealed record Chunk(string Id, byte[] Payload); + + private static List ReadChunks(byte[] image) { + Assert.That(image.AsSpan(image.Length - 12, 4).ToArray(), Is.EqualTo("NER5"u8.ToArray())); + var position = checked((int)BinaryPrimitives.ReadUInt64BigEndian(image.AsSpan(image.Length - 8))); + var footerOffset = image.Length - 12; + var chunks = new List(); + while (position <= footerOffset - 8) { + var id = System.Text.Encoding.ASCII.GetString(image, position, 4); + var length = checked((int)BinaryPrimitives.ReadUInt32BigEndian(image.AsSpan(position + 4, 4))); + Assert.That(position + 8L + length, Is.LessThanOrEqualTo(footerOffset)); + var payload = image.AsSpan(position + 8, length).ToArray(); + chunks.Add(new Chunk(id, payload)); + position += 8 + length; + if (id == "END!") + break; + } + Assert.That(position, Is.EqualTo(footerOffset)); + return chunks; + } + + private static void AssertCue(byte[] payload, int entryIndex, byte adrCtl, byte track, byte index, int lba) { + var offset = entryIndex * 8; + Assert.Multiple(() => { + Assert.That(payload[offset], Is.EqualTo(adrCtl)); + Assert.That(payload[offset + 1], Is.EqualTo(track)); + Assert.That(payload[offset + 2], Is.EqualTo(index)); + Assert.That(CueLba(payload, entryIndex), Is.EqualTo(lba)); + }); + } + + private static int CueLba(byte[] payload, int entryIndex) + => unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(payload.AsSpan(entryIndex * 8 + 4, 4))); +} From 279e26f1da6a36205499500df8ad402035d8d45a Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:28:48 +0200 Subject: [PATCH 144/165] test(onefs): pin unsupported integrity testing --- Compression.Tests/OneFs/OneFsDetectionTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Compression.Tests/OneFs/OneFsDetectionTests.cs b/Compression.Tests/OneFs/OneFsDetectionTests.cs index 5ed3d2f9c..05362dfff 100644 --- a/Compression.Tests/OneFs/OneFsDetectionTests.cs +++ b/Compression.Tests/OneFs/OneFsDetectionTests.cs @@ -154,6 +154,8 @@ public void WriteAndDestructiveMaintenanceCapabilities_RemainBlocked() { var descriptor = new OneFsFormatDescriptor(); Assert.Multiple(() => { + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanTest), Is.False, + "Opaque byte copying cannot validate an undocumented filesystem's integrity."); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); Assert.That(descriptor, Is.Not.InstanceOf()); From 5cec8a89af5ea325e9695e2d97f9b373b6457bce Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:29:31 +0200 Subject: [PATCH 145/165] Update NRG writer structure tests for DAO profile --- Compression.Tests/Nrg/NrgTests.cs | 37 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/Compression.Tests/Nrg/NrgTests.cs b/Compression.Tests/Nrg/NrgTests.cs index 162de18ba..d4d75934a 100644 --- a/Compression.Tests/Nrg/NrgTests.cs +++ b/Compression.Tests/Nrg/NrgTests.cs @@ -102,18 +102,45 @@ public void Read_Nrg_DetectsVersion2Footer() { } [Test, Category("HappyPath"), Category("RoundTrip")] - public void Create_WritesRealV2ChunkChain() { + public void Create_WritesRealV2DaoChunkChain() { var image = CreateNrg(("DATA.BIN", "nero-payload"u8.ToArray())); Assert.That(image.AsSpan(image.Length - 12, 4).ToArray(), Is.EqualTo("NER5"u8.ToArray())); var trailerOffset = checked((int)BinaryPrimitives.ReadUInt64BigEndian(image.AsSpan(image.Length - 8))); Assert.That(trailerOffset, Is.LessThan(image.Length - 12)); + Assert.That(trailerOffset % 2048, Is.Zero); var position = trailerOffset; - var etn2 = ReadChunk(image, ref position, "ETN2", 32); - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(etn2), Is.Zero); - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(etn2[8..]), Is.EqualTo((ulong)trailerOffset)); - Assert.That(etn2[19], Is.Zero, "cooked Mode 1"); + var cuex = ReadChunk(image, ref position, "CUEX", 32); + Assert.Multiple(() => { + Assert.That(cuex[0], Is.EqualTo(0x41)); + Assert.That(cuex[1], Is.EqualTo(0x00)); + Assert.That(cuex[2], Is.EqualTo(0x00)); + Assert.That(unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(cuex[4..])), Is.EqualTo(-150)); + Assert.That(cuex[9], Is.EqualTo(0x01)); + Assert.That(cuex[10], Is.EqualTo(0x00)); + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex[12..]), Is.Zero); + Assert.That(cuex[17], Is.EqualTo(0x01)); + Assert.That(cuex[18], Is.EqualTo(0x01)); + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex[20..]), Is.Zero); + Assert.That(cuex[25], Is.EqualTo(0xAA)); + Assert.That(cuex[26], Is.EqualTo(0x01)); + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex[28..]), Is.EqualTo((uint)(trailerOffset / 2048))); + }); + + var daox = ReadChunk(image, ref position, "DAOX", 64); + var track = daox[22..]; + Assert.Multiple(() => { + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(daox), Is.EqualTo(64)); + Assert.That(daox[20], Is.EqualTo(1)); + Assert.That(daox[21], Is.EqualTo(1)); + Assert.That(BinaryPrimitives.ReadUInt16BigEndian(track[12..]), Is.EqualTo(2048)); + Assert.That(track[14], Is.Zero, "cooked Mode 1"); + Assert.That(BinaryPrimitives.ReadUInt16BigEndian(track[16..]), Is.EqualTo(1)); + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track[18..]), Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track[26..]), Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track[34..]), Is.EqualTo((ulong)trailerOffset)); + }); var sinf = ReadChunk(image, ref position, "SINF", 4); Assert.That(BinaryPrimitives.ReadUInt32BigEndian(sinf), Is.EqualTo(1)); From b84e6013d6ce0becf9a8ef269c70da64b30ab565 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:29:38 +0200 Subject: [PATCH 146/165] CI: capture ReFS build diagnostics --- .github/workflows/refs-diagnostic.yml | 57 +++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/refs-diagnostic.yml diff --git a/.github/workflows/refs-diagnostic.yml b/.github/workflows/refs-diagnostic.yml new file mode 100644 index 000000000..3470908a9 --- /dev/null +++ b/.github/workflows/refs-diagnostic.yml @@ -0,0 +1,57 @@ +name: ReFS diagnostic + +on: + push: + branches: [refs-block-refcount-clone] + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Clone PNGCrushCS sibling + run: git clone --depth 1 https://github.com/Hawkynt/PNGCrushCS.git "$GITHUB_WORKSPACE/../PNGCrushCS" + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + dotnet-quality: preview + - name: Pre-build sibling + shell: bash + run: | + set -euo pipefail + cd "$GITHUB_WORKSPACE/../PNGCrushCS" + REQUIRED="Compression.Core FileFormat.Core FileFormat.Bmp FileFormat.Apng FileFormat.Tiff FileFormat.BigTiff FileFormat.Mng FileFormat.Fli FileFormat.Dcx FileFormat.Icns FileFormat.Mpo FileFormat.Jpeg FileFormat.Png" + locate() { + local proj="$1" + if [ -f "$proj/$proj.csproj" ]; then echo "$proj"; return 0; fi + if [ -f "FileFormats/$proj/$proj.csproj" ]; then echo "FileFormats/$proj"; return 0; fi + return 1 + } + for proj in $REQUIRED; do + if ! locate "$proj" >/dev/null; then exit 0; fi + done + for proj in $REQUIRED; do + dir="$(locate "$proj")" + dotnet build "$dir/$proj.csproj" --configuration Release --nologo + done + - name: Restore + run: dotnet restore CompressionWorkbench.slnx + - name: Build and capture diagnostics + id: build + shell: bash + continue-on-error: true + run: | + set -o pipefail + dotnet build CompressionWorkbench.slnx --configuration Release --no-restore 2>&1 | tee refs-build.log + - uses: actions/upload-artifact@v4 + with: + name: refs-build-log + path: refs-build.log + - name: Propagate build result + if: steps.build.outcome != 'success' + run: exit 1 From a420459aa9500c5524d41e199edc39d3fc5d67d7 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:34:00 +0200 Subject: [PATCH 147/165] Add credential-aware archive mutation options --- Compression.Registry/ArchiveMutationOptions.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Compression.Registry/ArchiveMutationOptions.cs diff --git a/Compression.Registry/ArchiveMutationOptions.cs b/Compression.Registry/ArchiveMutationOptions.cs new file mode 100644 index 000000000..fb8463249 --- /dev/null +++ b/Compression.Registry/ArchiveMutationOptions.cs @@ -0,0 +1,14 @@ +namespace Compression.Registry; + +/// +/// Per-operation options for mutating an existing archive or filesystem image. +/// +/// +/// Mutation credentials belong to the operation, not to the input entries. Keeping +/// them here avoids format-specific ambient state and lets encrypted containers +/// expose the same add/replace/remove contract as unencrypted formats. +/// +public sealed class ArchiveMutationOptions { + /// Password or passphrase required to unlock the existing container. + public string? Password { get; init; } +} From 087ca160080c96f1c7c9fccafaa5e67fbfa370e8 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:34:36 +0200 Subject: [PATCH 148/165] Add credential-aware mutation overloads --- Compression.Registry/IArchiveModifiable.cs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Compression.Registry/IArchiveModifiable.cs b/Compression.Registry/IArchiveModifiable.cs index 323ec77ea..c0876df2c 100644 --- a/Compression.Registry/IArchiveModifiable.cs +++ b/Compression.Registry/IArchiveModifiable.cs @@ -39,6 +39,17 @@ void Add(Stream archive, IReadOnlyList inputs) { }); } + /// Adds or replaces files while supplying operation-scoped credentials. + /// + /// The default deliberately delegates to the legacy overload so every existing + /// unencrypted modifier keeps its behavior unchanged. Password-aware formats + /// override this overload and consume . + /// + void Add(Stream archive, IReadOnlyList inputs, ArchiveMutationOptions options) { + ArgumentNullException.ThrowIfNull(options); + this.Add(archive, inputs); + } + /// /// Removes the named entries from an existing instance. Passing every entry name yields an /// empty container/image where the format permits one. @@ -60,4 +71,14 @@ void Remove(Stream archive, string[] entryNames) { } }); } + + /// Removes entries while supplying operation-scoped credentials. + /// + /// The default delegates to the legacy overload; formats that need a password to + /// locate or rewrite encrypted metadata can override this overload independently. + /// + void Remove(Stream archive, string[] entryNames, ArchiveMutationOptions options) { + ArgumentNullException.ThrowIfNull(options); + this.Remove(archive, entryNames); + } } From 5b15f23af205b1f5d6980ef3a80805b9cc5e232d Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:34:53 +0200 Subject: [PATCH 149/165] Expose streaming passphrase validation for eCryptfs mutation --- FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs b/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs index 8d4db9272..c952de2f2 100644 --- a/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs +++ b/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs @@ -76,6 +76,16 @@ public byte[] ExtractContent(string password) { return output.ToArray(); } + /// + /// Validates a passphrase against the authentication-token packet without + /// materializing plaintext in memory. The ciphertext is streamed through the + /// normal decoder into , so the same key and extent + /// path used by extraction is exercised before a destructive mutation starts. + /// + /// The passphrase does not match this lower file. + public void ValidatePassword(string password) + => EcryptfsCodec.DecryptTo(this._stream, Stream.Null, this._header, password); + public void Dispose() { // The reader never owns the caller's stream. } From 1c3cef656a917ec15d7aaafa4cd3a36d6b34932b Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:35:33 +0200 Subject: [PATCH 150/165] Promote eCryptfs to credential-aware R/W --- .../EcryptfsFormatDescriptor.cs | 83 ++++++++++++++++++- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs b/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs index 3cb42e901..a6fcb6bf0 100644 --- a/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs +++ b/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs @@ -20,9 +20,9 @@ public sealed class EcryptfsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, + IArchiveModifiable, IArchiveShrinkable, - IWipeEmpty, - IArchivePurgeable { + IWipeEmpty { public string Id => "Ecryptfs"; public string DisplayName => "eCryptfs"; @@ -32,6 +32,7 @@ public sealed class EcryptfsFormatDescriptor : FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate + | FormatCapabilities.CanModify | FormatCapabilities.SupportsPassword; public string DefaultExtension => ".ecryptfs"; @@ -54,7 +55,7 @@ public sealed class EcryptfsFormatDescriptor : public string? TarCompressionFormatId => null; public AlgorithmFamily Family => AlgorithmFamily.Archive; - public string Description => "Linux eCryptfs per-file lower-file encryption container."; + public string Description => "Linux eCryptfs per-file lower-file encryption container — passphrase AES read/create/replace/remove."; public List List(Stream stream, string? password) { using var reader = new EcryptfsReader(stream); @@ -98,6 +99,79 @@ public void Create(Stream output, IReadOnlyList inputs, Format EcryptfsCodec.Create(output, content, options.Password, EcryptfsCodec.ResolveKeySize(method)); } + /// + /// Credential-free mutation cannot safely replace encrypted plaintext. Callers + /// must use the + /// overload instead of relying on ambient or format-specific password state. + /// + void IArchiveModifiable.Add(Stream archive, IReadOnlyList inputs) + => throw new InvalidOperationException( + "eCryptfs replacement requires a passphrase; use the credential-aware IArchiveModifiable.Add overload."); + + /// + /// Replaces the one logical upper-file payload after validating the existing + /// lower file with the supplied passphrase. A complete replacement lower file + /// is staged before the caller's stream is changed, so a bad passphrase or + /// writer failure leaves the original bytes untouched. + /// + void IArchiveModifiable.Add( + Stream archive, + IReadOnlyList inputs, + ArchiveMutationOptions options + ) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + ArgumentNullException.ThrowIfNull(options); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("eCryptfs mutation requires a readable, writable, seekable stream.", nameof(archive)); + if (options.Password is null) + throw new ArgumentException("A passphrase is required to modify an eCryptfs lower file.", nameof(options)); + if (inputs.Any(i => i.IsDirectory)) + throw new NotSupportedException("An eCryptfs lower file represents one regular file; directories cannot be added."); + + var files = inputs.Where(i => !i.IsDirectory).ToList(); + if (files.Count != 1) + throw new NotSupportedException("eCryptfs replacement requires exactly one regular-file payload."); + + archive.Position = 0; + int keySize; + using (var reader = new EcryptfsReader(archive)) { + reader.ValidatePassword(options.Password); + keySize = reader.CipherDescription switch { + "AES-128-CBC" => 16, + "AES-192-CBC" => 24, + "AES-256-CBC" => 32, + _ => throw new NotSupportedException($"Cannot rewrite eCryptfs cipher '{reader.CipherDescription}'."), + }; + } + + var content = files[0].ReadContent(); + using var staged = new MemoryStream(); + EcryptfsCodec.Create(staged, content, options.Password, keySize); + + staged.Position = 0; + archive.Position = 0; + archive.SetLength(0); + staged.CopyTo(archive); + archive.Position = 0; + } + + /// + /// Removes the single logical payload. This does not require the FEK: the + /// plaintext length can be set to zero and ciphertext extents discarded while + /// retaining valid authentication-token metadata. + /// + void IArchiveModifiable.Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(entryNames); + if (entryNames.Any(IsContentName)) + EcryptfsCodec.Purge(archive); + } + + void IArchiveModifiable.Remove(Stream archive, string[] entryNames, ArchiveMutationOptions options) { + ArgumentNullException.ThrowIfNull(options); + ((IArchiveModifiable)this).Remove(archive, entryNames); + } + /// /// Clears metadata padding and bytes beyond the canonical encrypted extent set. /// Ciphertext inside the final live extent is never touched: plaintext tail @@ -118,4 +192,7 @@ public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipe /// to zero and truncate all ciphertext extents while retaining valid key metadata. /// public void Purge(Stream archive) => EcryptfsCodec.Purge(archive); + + private static bool IsContentName(string name) + => Path.GetFileName(name.Replace('\\', '/')).Equals("content.bin", StringComparison.OrdinalIgnoreCase); } From f17fbe61ceb88ff117a39d529306e4fa17050fa9 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:35:59 +0200 Subject: [PATCH 151/165] Assert eCryptfs credential-aware R/W capability --- Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs b/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs index 4aa01d4ca..88c66693b 100644 --- a/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs +++ b/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs @@ -17,7 +17,7 @@ private static byte[] CreateImage(byte[] content, string password = "foo", strin } [Test, Category("HappyPath")] - public void Descriptor_AdvertisesPasswordAwareWorm_WithoutFakeMagic() { + public void Descriptor_AdvertisesPasswordAwareRw_WithoutFakeMagic() { var descriptor = new EcryptfsFormatDescriptor(); Assert.That(descriptor.Id, Is.EqualTo("Ecryptfs")); Assert.That(descriptor.DisplayName, Is.EqualTo("eCryptfs")); @@ -25,9 +25,9 @@ public void Descriptor_AdvertisesPasswordAwareWorm_WithoutFakeMagic() { Assert.That(descriptor.MagicSignatures, Is.Empty, "The eCryptfs marker is a relation between two random words, not fixed bytes."); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.SupportsPassword), Is.True); - Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False, - "The generic mutation API carries no passphrase, so plaintext replacement cannot be exposed honestly."); + Assert.That(descriptor, Is.InstanceOf()); Assert.That(descriptor, Is.InstanceOf()); Assert.That(descriptor, Is.InstanceOf()); Assert.That(descriptor, Is.InstanceOf()); From 560aa13c5d73218a9a3041fec3fe3a2d753b23c2 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:36:33 +0200 Subject: [PATCH 152/165] Test credential-aware eCryptfs replacement and removal --- .../Ecryptfs/EcryptfsStubBehaviorTests.cs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs b/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs index 8cc4651d2..5c978528c 100644 --- a/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs +++ b/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs @@ -38,6 +38,74 @@ public void CreateThenExtract_RoundTripsAesPassphraseProfiles(string method) { Assert.That(reader.ExtractContent("correct horse"), Is.EqualTo(content)); } + [Test, Category("RoundTrip")] + public void CredentialAwareAdd_ReplacesPayload_AndPreservesCipherProfile() { + const string password = "correct horse"; + var replacement = Enumerable.Range(0, 12001).Select(i => (byte)(i * 41 + 3)).ToArray(); + var descriptor = new EcryptfsFormatDescriptor(); + var modifier = (IArchiveModifiable)descriptor; + using var image = new MemoryStream(CreateImage([1, 2, 3, 4, 5], password, "aes256")); + + image.Position = 0; + string? signatureBefore; + using (var before = new EcryptfsReader(image)) { + signatureBefore = before.PassphraseSignature; + Assert.That(before.CipherDescription, Is.EqualTo("AES-256-CBC")); + } + + image.Position = 0; + modifier.Add( + image, + [ArchiveInputInfo.InMemory("content.bin", replacement)], + new ArchiveMutationOptions { Password = password }); + + image.Position = 0; + using var reader = new EcryptfsReader(image); + Assert.That(reader.CipherDescription, Is.EqualTo("AES-256-CBC")); + Assert.That(reader.PassphraseSignature, Is.EqualTo(signatureBefore)); + Assert.That(reader.ExtractContent(password), Is.EqualTo(replacement)); + } + + [Test, Category("Sad")] + public void CredentialAwareAdd_WrongPassphrase_LeavesOriginalByteIdentical() { + var descriptor = new EcryptfsFormatDescriptor(); + var modifier = (IArchiveModifiable)descriptor; + using var image = new MemoryStream(CreateImage(Enumerable.Range(0, 5000).Select(i => (byte)i).ToArray())); + var before = image.ToArray(); + + image.Position = 0; + Assert.Throws(() => modifier.Add( + image, + [ArchiveInputInfo.InMemory("content.bin", [9, 8, 7, 6])], + new ArchiveMutationOptions { Password = "wrong horse" })); + + Assert.That(image.ToArray(), Is.EqualTo(before)); + } + + [Test, Category("Sad")] + public void CredentialFreeAdd_IsRejectedInsteadOfGuessingPassword() { + var modifier = (IArchiveModifiable)new EcryptfsFormatDescriptor(); + using var image = new MemoryStream(CreateImage([1, 2, 3])); + Assert.Throws(() => modifier.Add( + image, + [ArchiveInputInfo.InMemory("content.bin", [4, 5, 6])])); + } + + [Test, Category("HappyPath")] + public void Remove_ContentBin_LeavesValidEmptyLowerFile() { + var descriptor = new EcryptfsFormatDescriptor(); + var modifier = (IArchiveModifiable)descriptor; + using var image = new MemoryStream(CreateImage(Enumerable.Repeat((byte)0x6D, 5000).ToArray())); + + modifier.Remove(image, ["content.bin"]); + + Assert.That(image.Length, Is.EqualTo(8192)); + image.Position = 0; + Assert.That(descriptor.List(image, null), Is.Empty); + using var reader = new EcryptfsReader(image); + reader.ValidatePassword("correct horse"); + } + [Test, Category("Sad")] public void Extract_WrongPassphrase_IsRejectedBeforePayloadDecryption() { using var image = new MemoryStream(CreateImage([1, 2, 3, 4]), writable: false); From ae88faee12a049192e8487022884a6b1b71e2277 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:37:57 +0200 Subject: [PATCH 153/165] Add xattr-safe ext-backed Gluster shrink --- .../GlusterFsFormatDescriptor.cs | 83 ++++++++++++++++--- 1 file changed, 73 insertions(+), 10 deletions(-) diff --git a/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs b/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs index 29949dd9a..38eebd78a 100644 --- a/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs +++ b/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs @@ -1,12 +1,15 @@ #pragma warning disable CS1591 +using System.Buffers.Binary; using Compression.Registry; using Compression.Registry.Streaming; +using FileSystem.Ext; using static Compression.Registry.FormatHelpers; namespace FileSystem.GlusterFs; /// -/// Read-only single-brick view for GlusterFS backing-store images. +/// Read-only single-brick view for GlusterFS backing-store images, with a +/// conservative backing-store shrink operation where it is provably metadata-safe. /// /// GlusterFS has no independent block format: a volume is a logical /// collection of bricks and each brick is an export directory on an ordinary @@ -21,12 +24,19 @@ namespace FileSystem.GlusterFs; /// 0xCAFE5BAB probe convention is deliberately not recognised as GlusterFS. /// /// The native XFS/ext layers can now read Gluster xattrs and conservatively -/// mutate the common short-form/in-inode storage cases. This descriptor remains -/// read-only until every xattr storage form that a maintenance operation can +/// mutate the common short-form/in-inode storage cases. General archive mutation +/// remains disabled until every xattr storage form that a maintenance operation can /// encounter (including ext external blocks/EA inodes and XFS leaf/btree/remote /// attributes) is preserved. Advertising a rebuild-based mutation before then /// could silently discard trusted.gfid or trusted.glusterfs.* state. /// +/// Shrink is the deliberate exception. For ext-backed bricks the native +/// in-place shrinker chooses its boundary from the filesystem allocation bitmap; +/// every allocated xattr block therefore pins the boundary just like file data and +/// is preserved byte-for-byte. XFS-backed bricks copy through unchanged because the +/// current XFS shrink path may rebuild the image. This is a backing-store geometry +/// operation only; it is not Gluster's remove-brick command. +/// /// Gluster volume operations such as rebalance, fix-layout, and remove-brick /// are explicitly outside this single-image abstraction. They coordinate multiple /// bricks and belong to a live cluster/volume control plane, not an offline brick @@ -42,7 +52,7 @@ namespace FileSystem.GlusterFs; /// https://github.com/gluster/glusterfs — canonical GlusterFS implementation, dual GPLv2/LGPLv3+ /// /// -public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { +public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveShrinkable { /// /// Gets the id. @@ -94,12 +104,12 @@ public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveForma /// Gets the description. /// public string Description => - "GlusterFS single-brick R/O backing-store view via XFS/ext delegation. GlusterFS has no " + - "standalone image format; the supplied .gluster image is treated as one brick's backing " + - "filesystem. The .glusterfs GFID index is hidden from normal listings. Native backing " + - "accessors can read trusted.gfid/trusted.glusterfs.* and mutate inline/short-form xattrs, " + - "but archive mutations stay disabled until external/leaf/btree xattr forms are safe. " + - "Cluster rebalance, fix-layout, and remove-brick are intentionally out of scope."; + "GlusterFS single-brick R/O backing-store view via XFS/ext delegation. Native backing " + + "accessors read trusted.gfid/trusted.glusterfs.* and mutate inline/short-form xattrs. " + + "General writes, defrag, wipe, layout and purge stay disabled until all xattr storage " + + "forms are safe. Shrink-to-fit is supported for ext-backed bricks via the allocation " + + "bitmap; XFS-backed bricks copy through unchanged. Cluster rebalance, fix-layout and " + + "remove-brick are intentionally out of scope."; /// /// Lists the entries in the supplied brick backing-store image. @@ -131,4 +141,57 @@ Stream IArchiveFormatOperations.OpenEntry(Stream archive, string entryName, stri var data = reader.Extract(entry); return new BoundedEntryStream(new MemoryStream(data, writable: false), data.Length, leaveOpen: false); } + + /// + /// Shrinks an ext-backed brick to its highest allocated block without rebuilding + /// file or xattr metadata. XFS-backed bricks are copied unchanged. + /// + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("GlusterFS shrink requires a readable, seekable input.", nameof(input)); + if (!output.CanRead || !output.CanWrite || !output.CanSeek) + throw new ArgumentException("GlusterFS shrink requires a readable, writable, seekable output.", nameof(output)); + + Stream target; + if (ReferenceEquals(input, output)) { + target = output; + } else { + input.Position = 0; + output.Position = 0; + output.SetLength(0); + input.CopyTo(output); + output.Flush(); + target = output; + } + + if (!IsExtBacking(target)) { + target.Position = 0; + return; + } + + target.Position = 0; + try { + ExtInPlaceShrinker.ShrinkToFit(target); + } catch (NotSupportedException) { + // Some valid ext geometries cannot be reduced by the conservative in-place + // shrinker. Leaving the already-copied image unchanged is the safe result. + } + target.Position = 0; + } + + private static bool IsExtBacking(Stream image) { + const long magicOffset = 1024 + 56; + if (image.Length < magicOffset + sizeof(ushort)) return false; + var original = image.Position; + Span magic = stackalloc byte[sizeof(ushort)]; + try { + image.Position = magicOffset; + image.ReadExactly(magic); + return BinaryPrimitives.ReadUInt16LittleEndian(magic) == 0xEF53; + } finally { + image.Position = original; + } + } } From 7756dba1b25b2faed983c21139d18a1812d89963 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:38:10 +0200 Subject: [PATCH 154/165] Exercise credential-aware filesystem write round trips --- .../FilesystemWriteRoundTripTests.cs | 45 +++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs b/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs index 1b40350f6..f128a7769 100644 --- a/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs +++ b/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs @@ -43,6 +43,7 @@ public sealed class FilesystemWriteRoundTripTests { private const string ProbeStem = "PROBE"; private const string Seed = "SEED.BIN"; private const string SeedStem = "SEED"; + private const string TestPassword = "cwb-filesystem-roundtrip"; /// /// Ids whose write claim this fixture cannot exercise, and why. Each entry is a @@ -156,7 +157,8 @@ private void EditByName(string formatId, IArchiveModifiable modifier, byte[] ima var added = Payload(3, 512); byte[] withProbe; try { - withProbe = Mutate(formatId, image, m => modifier.Add(m, [ArchiveInputInfo.InMemory(Probe, added)]), "Add"); + withProbe = Mutate(formatId, image, + m => Add(modifier, formatId, m, [ArchiveInputInfo.InMemory(Probe, added)]), "Add"); } catch (NotSupportedException) { this.EditByOwnEntry(formatId, modifier, image); return; @@ -170,9 +172,9 @@ private void EditByName(string formatId, IArchiveModifiable modifier, byte[] ima // and the bare leaf second. Both together, in one call, is what a strict // remover reads as "one of these is missing". var stored = Names(formatId, withProbe).First(n => Matches(n, ProbeStem)); - var afterRemove = Mutate(formatId, withProbe, m => modifier.Remove(m, [stored]), "Remove"); + var afterRemove = Mutate(formatId, withProbe, m => Remove(modifier, formatId, m, [stored]), "Remove"); if (Names(formatId, afterRemove).Any(n => Matches(n, ProbeStem)) && Leaf(stored) != stored) - afterRemove = Mutate(formatId, withProbe, m => modifier.Remove(m, [Leaf(stored)]), "Remove"); + afterRemove = Mutate(formatId, withProbe, m => Remove(modifier, formatId, m, [Leaf(stored)]), "Remove"); Assert.That(Names(formatId, afterRemove).Any(n => Matches(n, ProbeStem)), Is.False, $"{formatId}: '{stored}' is still listed after Remove."); this.AssertReadsBack(formatId, afterRemove, SeedStem, seed, "after remove (survivor)"); @@ -193,7 +195,7 @@ private void EditByOwnEntry(string formatId, IArchiveModifiable modifier, byte[] var ops = (IArchiveFormatOperations)FormatRegistry.GetArchiveOps(formatId)!; List entries; using (var stream = new MemoryStream(image, writable: false)) - entries = ops.List(stream, null).Where(e => !e.IsDirectory).ToList(); + entries = ops.List(stream, PasswordFor(formatId)).Where(e => !e.IsDirectory).ToList(); // metadata.ini and its kin are rendered from the header, not stored, so // rewriting one asks the format to accept a report of itself as input. @@ -217,7 +219,7 @@ private void EditByOwnEntry(string formatId, IArchiveModifiable modifier, byte[] byte[] edited; try { edited = Mutate(formatId, image, - m => modifier.Add(m, [ArchiveInputInfo.InMemory(candidate.Name, replacement)]), "Add"); + m => Add(modifier, formatId, m, [ArchiveInputInfo.InMemory(candidate.Name, replacement)]), "Add"); } catch (Exception ex) when (ex is NotSupportedException or ArgumentException or InvalidDataException) { refusals.Add($"{candidate.Name}: {ex.Message}"); continue; @@ -291,7 +293,7 @@ public void UnitAddressed_PointAtFixturesThatExist() { private static byte[]? Create(string formatId, IReadOnlyList inputs) { if (FormatRegistry.GetArchiveOps(formatId) is not IArchiveCreatable creator) return null; using var image = new MemoryStream(); - creator.Create(image, inputs, new FormatCreateOptions()); + creator.Create(image, inputs, new FormatCreateOptions { Password = PasswordFor(formatId) }); return image.Length == 0 ? null : image.ToArray(); } @@ -308,9 +310,36 @@ private static byte[] Mutate(string formatId, byte[] image, Action private static List Names(string formatId, byte[] image) { var ops = (IArchiveFormatOperations)FormatRegistry.GetArchiveOps(formatId)!; using var stream = new MemoryStream(image, writable: false); - return ops.List(stream, null).Where(e => !e.IsDirectory).Select(e => e.Name).ToList(); + return ops.List(stream, PasswordFor(formatId)).Where(e => !e.IsDirectory).Select(e => e.Name).ToList(); } + private static void Add( + IArchiveModifiable modifier, + string formatId, + Stream archive, + IReadOnlyList inputs + ) { + var password = PasswordFor(formatId); + if (password is null) + modifier.Add(archive, inputs); + else + modifier.Add(archive, inputs, new ArchiveMutationOptions { Password = password }); + } + + private static void Remove(IArchiveModifiable modifier, string formatId, Stream archive, string[] entryNames) { + var password = PasswordFor(formatId); + if (password is null) + modifier.Remove(archive, entryNames); + else + modifier.Remove(archive, entryNames, new ArchiveMutationOptions { Password = password }); + } + + private static string? PasswordFor(string formatId) + => (FormatRegistry.GetArchiveOps(formatId) as IFormatDescriptor)?.Capabilities + .HasFlag(FormatCapabilities.SupportsPassword) == true + ? TestPassword + : null; + private void AssertReadsBack(string formatId, byte[] image, string stem, byte[] expected, string step) { var extracted = this.ExtractAll(formatId, image); var got = extracted.FirstOrDefault(e => Matches(e.Leaf, stem)).Bytes; @@ -343,7 +372,7 @@ private void AssertEveryEntryExtracts(string formatId, byte[] image, string step try { var ops = (IArchiveFormatOperations)FormatRegistry.GetArchiveOps(formatId)!; using (var stream = new MemoryStream(image, writable: false)) - ops.Extract(stream, outDir, null, null); + ops.Extract(stream, outDir, PasswordFor(formatId), null); return Directory.EnumerateFiles(outDir, "*", SearchOption.AllDirectories) .Select(path => (Path.GetFileName(path), File.ReadAllBytes(path))) .ToList(); From 8f6cb8b259e053b111b6914a1e02b46826c29f30 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:38:39 +0200 Subject: [PATCH 155/165] Test xattr-safe Gluster shrink behavior --- .../GlusterFs/GlusterFsDetectionTests.cs | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs b/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs index cdd2ef183..98cfce84b 100644 --- a/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs +++ b/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs @@ -56,6 +56,7 @@ public void Descriptor_UsesExtensionOnlyDetectionAndAdvertisesReadOnlyBrickView( Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.SupportsDirectories), Is.True); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + Assert.That(descriptor, Is.InstanceOf()); }); } @@ -117,6 +118,41 @@ public void XfsBackingStore_ExtractsThroughDescriptor() { } } + [Test, Category("RoundTrip")] + public void ExtBackingStore_ShrinkToFit_PreservesPayloadAndGfid() { + var original = BuildExtBrick(); + var descriptor = new GlusterFsFormatDescriptor(); + using var input = new MemoryStream(original, writable: false); + using var output = new MemoryStream(); + + descriptor.Shrink(input, output); + + Assert.That(output.Length, Is.LessThan(original.LongLength)); + output.Position = 0; + using var reader = new GlusterFsReader(output); + var entry = reader.Entries.Single(candidate => candidate.Name == "brick/data/hello.txt"); + Assert.Multiple(() => { + Assert.That(Encoding.UTF8.GetString(reader.Extract(entry)), Is.EqualTo("hello from ext brick")); + Assert.That(reader.ReadExtendedAttributes(entry)["trusted.gfid"], Is.EqualTo(Gfid)); + }); + } + + [Test, Category("RoundTrip")] + public void XfsBackingStore_ShrinkCopiesThroughWithoutRebuild() { + var original = BuildXfsBrick(); + var descriptor = new GlusterFsFormatDescriptor(); + using var input = new MemoryStream(original, writable: false); + using var output = new MemoryStream(); + + descriptor.Shrink(input, output); + + Assert.That(output.ToArray(), Is.EqualTo(original)); + output.Position = 0; + using var reader = new GlusterFsReader(output); + var entry = reader.Entries.Single(candidate => candidate.Name == "brick/data/hello.txt"); + Assert.That(reader.ReadExtendedAttributes(entry)["trusted.gfid"], Is.EqualTo(Gfid)); + } + [Test, Category("Regression")] public void WorkbenchProbeMagic_IsNoLongerAcceptedAsGlusterFs() { byte[] oldSyntheticProbe = [0xCA, 0xFE, 0x5B, 0xAB, 0, 0, 0, 0]; @@ -128,7 +164,7 @@ public void WorkbenchProbeMagic_IsNoLongerAcceptedAsGlusterFs() { } [Test, Category("Regression")] - public void Descriptor_DoesNotAdvertiseMutationsUntilAllXattrStorageFormsAreSafe() { + public void Descriptor_DoesNotAdvertiseUnsafeMutations() { var descriptor = new GlusterFsFormatDescriptor(); Assert.Multiple(() => { @@ -136,7 +172,6 @@ public void Descriptor_DoesNotAdvertiseMutationsUntilAllXattrStorageFormsAreSafe Assert.That(descriptor, Is.Not.InstanceOf()); Assert.That(descriptor, Is.Not.InstanceOf()); Assert.That(descriptor, Is.Not.InstanceOf()); - Assert.That(descriptor, Is.Not.InstanceOf()); Assert.That(descriptor, Is.Not.InstanceOf()); Assert.That(descriptor, Is.Not.InstanceOf()); }); From f4a1d69013fccf0c2a2f66e4b4ce9cdcc5717189 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:39:33 +0200 Subject: [PATCH 156/165] Make maintenance probes credential-aware --- .../Support/MaintenanceOperationProbe.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Compression.Tests/Support/MaintenanceOperationProbe.cs b/Compression.Tests/Support/MaintenanceOperationProbe.cs index 5b59b645e..c421059a3 100644 --- a/Compression.Tests/Support/MaintenanceOperationProbe.cs +++ b/Compression.Tests/Support/MaintenanceOperationProbe.cs @@ -22,6 +22,7 @@ namespace Compression.Tests.Support; /// internal static class MaintenanceOperationProbe { public const string ProbeName = "PROBE.BIN"; + private const string TestPassword = "cwb-maintenance-probe"; private static readonly byte[] ProbeData = CreateProbeData(); private static readonly string ProbeDigest = Digest(ProbeData); @@ -59,7 +60,7 @@ public static ProbeImage CreateImage(string formatId, string workDirectory) { var image = Path.Combine(workDirectory, "probe.img"); try { ArchiveOperations.Create(image, [new ArchiveInput(sourcePath, ProbeName)], - new CompressionOptions(), format, null); + new CompressionOptions { Password = PasswordFor(ops!) }, format, null); } catch (Exception ex) { Assert.Fail($"{formatId}: advertises a maintenance verb but neither creates the standard probe image " + $"nor declares the input inadmissible through IArchiveWriteConstraints: {ex.GetType().Name}: {ex.Message}"); @@ -93,15 +94,16 @@ public static void AssertProbeFilesAbsent(IArchiveFormatOperations ops, Stream i public static IReadOnlyList ListFiles(IArchiveFormatOperations ops, Stream image) { if (image.CanSeek) image.Position = 0; - return ops.List(image, null).Where(entry => !entry.IsDirectory).ToArray(); + return ops.List(image, PasswordFor(ops)).Where(entry => !entry.IsDirectory).ToArray(); } private static IReadOnlyList ProbeEntries(IArchiveFormatOperations ops, Stream image) { var matches = new List(); + var password = PasswordFor(ops); foreach (var entry in ListFiles(ops, image)) { try { if (image.CanSeek) image.Position = 0; - using var payload = ops.OpenEntry(image, entry.Name, null); + using var payload = ops.OpenEntry(image, entry.Name, password); if (Digest(payload) == ProbeDigest) matches.Add(entry); } catch { @@ -112,6 +114,11 @@ private static IReadOnlyList ProbeEntries(IArchiveFormatOperat return matches; } + private static string? PasswordFor(IArchiveFormatOperations ops) + => (ops as IFormatDescriptor)?.Capabilities.HasFlag(FormatCapabilities.SupportsPassword) == true + ? TestPassword + : null; + private static byte[] CreateProbeData() { var result = new byte[4096]; for (var i = 0; i < result.Length; ++i) From a38af5f474538c56f46490e35e68a622241a6f1a Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:42:03 +0200 Subject: [PATCH 157/165] Fix NRG profile-aware shrink build --- .../FileFormat.Nrg/NrgFormatDescriptor.cs | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs b/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs index e904a94fb..7d8f55b70 100644 --- a/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs +++ b/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs @@ -170,14 +170,35 @@ public void Shrink(Stream input, Stream output) { ArgumentNullException.ThrowIfNull(output); var profile = NrgStructureInspector.Inspect(input); if (!profile.IsSingleDataTrack) { + CopyThrough(input, output); + return; + } + + using var rebuilt = RebuildVerb.CreateScratchStream(); + var useRebuilt = false; + try { + RebuildVerb.RebuildToStream(input, rebuilt, this, this); + useRebuilt = rebuilt.Length > 0 && rebuilt.Length < input.Length; + } catch { + useRebuilt = false; + } + + output.Position = 0; + output.SetLength(0); + if (useRebuilt) { + rebuilt.Position = 0; + rebuilt.CopyTo(output); + } else { input.Position = 0; - output.Position = 0; - output.SetLength(0); input.CopyTo(output); - return; } + } - ((IArchiveShrinkable)this).ShrinkDefault(input, output); + private static void CopyThrough(Stream input, Stream output) { + input.Position = 0; + output.Position = 0; + output.SetLength(0); + input.CopyTo(output); } private static void EnsureNamedFileMutationProfile(Stream archive) { From eea43b3c0ec7da681d2db00d9c0307063387951e Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:43:52 +0200 Subject: [PATCH 158/165] Correct NRG DAO TOC and pregap semantics --- FileFormats/FileFormat.Nrg/NrgWriter.cs | 32 +++++++++++++++++++------ 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/FileFormats/FileFormat.Nrg/NrgWriter.cs b/FileFormats/FileFormat.Nrg/NrgWriter.cs index 401c0c334..3af39d86a 100644 --- a/FileFormats/FileFormat.Nrg/NrgWriter.cs +++ b/FileFormats/FileFormat.Nrg/NrgWriter.cs @@ -34,7 +34,8 @@ public sealed record NrgTrackDefinition(NrgTrackMode Mode, byte[] Data) { /// /// Optional pregap bytes. When supplied they must contain exactly sectors, - /// or, when is zero, determine the pregap length themselves. + /// or, when is zero, determine the pregap length themselves. Raw framed + /// and subchannel modes require explicit bytes because an all-zero sector would not be valid framing. /// public byte[]? PregapData { get; init; } @@ -141,15 +142,16 @@ public static void Write(Stream output, NrgDiscDefinition disc) { private static List WriteTrackData(Stream output, NrgDiscDefinition disc) { var sessions = new List(disc.Sessions.Count); var nextTrackNumber = 1; - var programLba = 0; var previousLeadOutLba = 0; for (var sessionIndex = 0; sessionIndex < disc.Sessions.Count; ++sessionIndex) { var definition = disc.Sessions[sessionIndex]; + var firstTrackPregap = GetPregapSectors(definition.Tracks[0], GetSectorSize(definition.Tracks[0].Mode)); int leadInLba; + int programLba; if (sessionIndex == 0) { leadInLba = -150; - programLba = 0; + programLba = -firstTrackPregap; } else { var previousLeadOutLength = sessionIndex == 1 ? FirstLeadOutSectors : LaterLeadOutSectors; leadInLba = checked(previousLeadOutLba + previousLeadOutLength); @@ -219,8 +221,7 @@ private static void WriteDaox(Stream output, SessionLayout session) { Encoding.ASCII.GetBytes(mcn, payload.AsSpan(4, 13)); payload[17] = 0; - payload[18] = session.Tracks.Any(static track => IsMode2(track.Definition.Mode)) ? (byte)0x20 : (byte)0x00; - payload[19] = 0x01; + BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(18, 2), GetTocType(session)); payload[20] = checked((byte)session.Tracks[0].TrackNumber); payload[21] = checked((byte)session.Tracks[^1].TrackNumber); @@ -242,6 +243,12 @@ private static void WriteDaox(Stream output, SessionLayout session) { WriteChunk(output, "DAOX"u8, payload); } + private static ushort GetTocType(SessionLayout session) { + if (session.Tracks.Any(static track => IsMode2(track.Definition.Mode))) + return 0x2001; + return session.Tracks.Any(static track => !IsAudio(track.Definition.Mode)) ? (ushort)0x0001 : (ushort)0x0000; + } + private static void WriteCueEntry(Span destination, byte adrCtl, byte track, byte index, int lba) { destination[0] = adrCtl; destination[1] = track; @@ -251,7 +258,7 @@ private static void WriteCueEntry(Span destination, byte adrCtl, byte trac } private static byte AdrCtl(NrgTrackDefinition track) { - var control = track.Mode is NrgTrackMode.Audio or NrgTrackMode.AudioWithSubchannel ? 0x00 : 0x40; + var control = IsAudio(track.Mode) ? 0x00 : 0x40; if (track.CopyPermitted) control |= 0x20; return checked((byte)(control | 0x01)); @@ -266,8 +273,11 @@ private static byte ToBcd(int value) { private static int GetPregapSectors(NrgTrackDefinition track, int sectorSize) { if (track.PregapSectors < 0) throw new ArgumentOutOfRangeException(nameof(track.PregapSectors), "Pregap sector count cannot be negative."); - if (track.PregapData is not { Length: > 0 } bytes) + if (track.PregapData is not { Length: > 0 } bytes) { + if (track.PregapSectors != 0 && RequiresExplicitPregapData(track.Mode)) + throw new ArgumentException($"NRG {track.Mode} pregaps require explicit PregapData with valid sector framing.", nameof(track)); return track.PregapSectors; + } if (bytes.Length % sectorSize != 0) throw new ArgumentException("NRG pregap data must be an integral number of sectors.", nameof(track)); @@ -277,6 +287,10 @@ private static int GetPregapSectors(NrgTrackDefinition track, int sectorSize) { return fromBytes; } + private static bool RequiresExplicitPregapData(NrgTrackMode mode) + => mode is NrgTrackMode.Mode1Raw or NrgTrackMode.Mode2Raw or + NrgTrackMode.Mode1RawWithSubchannel or NrgTrackMode.AudioWithSubchannel or NrgTrackMode.Mode2RawWithSubchannel; + private static void ValidateDisc(NrgDiscDefinition disc) { if (disc.Sessions is null || disc.Sessions.Count == 0) throw new ArgumentException("An NRG disc must contain at least one session.", nameof(disc)); @@ -320,11 +334,15 @@ private static void ValidateIsrc(string? isrc) { throw new ArgumentException("NRG ISRC must contain exactly 12 printable ASCII characters.", nameof(isrc)); } + private static bool IsAudio(NrgTrackMode mode) + => mode is NrgTrackMode.Audio or NrgTrackMode.AudioWithSubchannel; + private static bool IsMode2(NrgTrackMode mode) => mode is NrgTrackMode.Mode2Form1 or NrgTrackMode.Mode2Form2 or NrgTrackMode.Mode2Raw or NrgTrackMode.Mode2RawWithSubchannel; private static void WriteZeros(Stream output, long length) { Span zeros = stackalloc byte[4096]; + zeros.Clear(); while (length > 0) { var count = (int)Math.Min(length, zeros.Length); output.Write(zeros[..count]); From c51bca08ef1737412414d01fbdfde70f045c3821 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:50:52 +0200 Subject: [PATCH 159/165] Harden TUX3 extent failure handling --- .../FileSystem.Tux3/Tux3FormatDescriptor.cs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs b/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs index 51749513f..f8a417f26 100644 --- a/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs +++ b/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs @@ -94,36 +94,35 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? /// /// Enumerates the provable byte layout without guessing at undecoded TUX3 allocation state. /// The declared native volume is reserved wholesale; only trailing bytes outside it are free. + /// Invalid or arithmetically unrepresentable volume metadata reserves the complete physical image. /// public IEnumerable EnumerateExtents(Stream image) { ArgumentNullException.ThrowIfNull(image); if (!image.CanRead || !image.CanSeek) return []; + var imageLength = image.Length; try { using var reader = new Tux3Reader(image); if (!TryGetDeclaredVolumeLength(reader, out var volumeLength)) - return []; + return ReserveWholeImage(imageLength); - var imageLength = image.Length; if (volumeLength >= imageLength) - return imageLength == 0 - ? [] - : [new DefragBlockInfo(0, imageLength, DefragBlockKind.MetadataReserved, "TUX3 volume (allocation map unresolved)")]; + return ReserveWholeImage(imageLength); return [ new DefragBlockInfo(0, volumeLength, DefragBlockKind.MetadataReserved, "TUX3 volume (allocation map unresolved)"), new DefragBlockInfo(volumeLength, imageLength - volumeLength, DefragBlockKind.Free, "Trailing bytes outside TUX3 volume"), ]; } catch (InvalidDataException) { - return []; + return ReserveWholeImage(imageLength); } catch (IOException) { - return []; + return ReserveWholeImage(imageLength); } } /// - /// Removes only bytes beyond the volume size declared by volblocks << blockbits. + /// Removes only bytes beyond the volume size declared by volblocks * blocksize. /// Malformed, truncated, or arithmetically invalid images are copied through unchanged. /// public void Shrink(Stream input, Stream output) { @@ -172,6 +171,11 @@ private static bool TryGetDeclaredVolumeLength(Tux3Reader reader, out long lengt return true; } + private static IEnumerable ReserveWholeImage(long imageLength) + => imageLength == 0 + ? [] + : [new DefragBlockInfo(0, imageLength, DefragBlockKind.MetadataReserved, "TUX3 image (volume boundary unresolved)")]; + private static void CopyPrefix(Stream input, Stream output, long count) { var buffer = new byte[128 * 1024]; while (count > 0) { From 12e1ab9540f8d7a45eab5d286a034784d2e3c9a2 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 10:51:40 +0200 Subject: [PATCH 160/165] Add adversarial TUX3 overflow regression --- Compression.Tests/Tux3/Tux3Tests.cs | 35 +++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/Compression.Tests/Tux3/Tux3Tests.cs b/Compression.Tests/Tux3/Tux3Tests.cs index db0cd2506..55cbda2cc 100644 --- a/Compression.Tests/Tux3/Tux3Tests.cs +++ b/Compression.Tests/Tux3/Tux3Tests.cs @@ -206,16 +206,37 @@ public void TruncatedDeclaredVolume_IsNeverTreatedAsFreeOrShrunk() { } [Test, Category("Regression")] - public void OverflowingDeclaredVolume_FailsClosed() { - var image = BuildNativeImage(volBlocks: ulong.MaxValue, blockBits: 12); + public void OverflowingDeclaredVolume_FailsClosedInsteadOfTrustingWrappedShift() { + const ushort blockBits = 12; + const ulong volBlocks = (1UL << 52) + 3; + const ulong wrappedLength = volBlocks << blockBits; + var image = BuildNativeImage(volBlocks: volBlocks, blockBits: blockBits); + image[^1] = 0xA5; var descriptor = new Tux3FormatDescriptor(); + Assert.That(wrappedLength, Is.EqualTo(3UL << blockBits), + "The regression fixture must wrap a mathematically >64-bit volume into a plausible small boundary."); + using var layoutStream = new MemoryStream(image, writable: false); - Assert.That(descriptor.EnumerateExtents(layoutStream), Is.Empty); + var extents = descriptor.EnumerateExtents(layoutStream).ToArray(); + Assert.That(extents, Has.Length.EqualTo(1)); + Assert.Multiple(() => { + Assert.That(extents[0].Offset, Is.Zero); + Assert.That(extents[0].Length, Is.EqualTo(image.LongLength)); + Assert.That(extents[0].Kind, Is.EqualTo(DefragBlockKind.MetadataReserved)); + }); - using var input = new MemoryStream(image, writable: false); - using var output = new MemoryStream(); - descriptor.Shrink(input, output); - Assert.That(output.ToArray(), Is.EqualTo(image)); + var wipeCopy = image.ToArray(); + using var wipeStream = new MemoryStream(wipeCopy, writable: true); + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(wipeStream, wipeClusterTips: false, wipeDeletedEntries: false); + Assert.Multiple(() => { + Assert.That(wiped, Is.Zero); + Assert.That(wipeCopy, Is.EqualTo(image)); + }); + + using var shrinkInput = new MemoryStream(image, writable: false); + using var shrinkOutput = new MemoryStream(); + descriptor.Shrink(shrinkInput, shrinkOutput); + Assert.That(shrinkOutput.ToArray(), Is.EqualTo(image)); } } From 2ea7c33ac8422eb84cbedbf89be16918cdbb27cb Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 13:20:42 +0200 Subject: [PATCH 161/165] # repair the integrated branches against each other Collisions that only appear once the branches sit side by side; no behaviour change intended. - Two branches each added an IArchiveModifiable overload, making every bare and .Remove ambiguous (CS0419). - UefiFvReader and four fixtures captured a ReadOnlySpan in a lambda. - NrgWriter allocated inside a loop (CA2014). - NRG reached RebuildVerb.CreateScratchStream from another assembly, where it was internal. - An external-oracle fixture imported System.ComponentModel for Win32Exception, making [Category] ambiguous against NUnit's. - CSO fixtures called a default interface member on the concrete descriptor. - A Lustre fixture's Math.Max(int, ushort) matched several overloads. - Two fixtures built fixed-capacity MemoryStreams for verbs that grow the image. # the shared maintenance probe encrypted archives the verbs cannot open Making the probe credential-aware meant every password-capable format got an encrypted probe -- but Defragment(Stream) and Purge take no credentials, so Zip, 7z, Rar, Ace, Arj, Sqx, Cb7 and Cbr all failed their round trips. The probe is unencrypted, falling back to a password only where creation refuses without one. * regenerate the filesystem support matrix from the live registry --- .../DiskImage/PartitionedDiskLister.cs | 4 +-- .../IArchiveWriteConstraints.cs | 2 +- Compression.Registry/RebuildVerb.cs | 2 +- .../Cso/CsoInPlaceModifyTests.cs | 2 +- Compression.Tests/Cso/CsoTests.cs | 6 ++-- .../Ecryptfs/EcryptfsStubBehaviorTests.cs | 6 +++- .../FirmwareHex/TiTxtMaintenanceTests.cs | 10 ++++++- .../Lustre/LustreDetectionTests.cs | 2 +- Compression.Tests/Nrg/NrgAuthoringTests.cs | 13 +++++---- Compression.Tests/Nrg/NrgTests.cs | 28 +++++++++---------- .../Refs/RefsBlockCloneExternalTests.cs | 1 + .../Support/MaintenanceOperationProbe.cs | 13 +++++++-- Compression.UI/Views/DefragmentWindow.xaml.cs | 2 +- .../MacriumFormatDescriptor.cs | 4 +-- FileFormats/FileFormat.Nrg/NrgWriter.cs | 4 ++- FileFormats/FileFormat.UefiFv/UefiFvReader.cs | 10 +++++-- .../FileSystem.SysV/SysVFormatDescriptor.cs | 2 +- .../FileSystem.SysV/SysVInPlaceModifier.cs | 2 +- Hawkynt.FileFormats.FileSystems/README.md | 14 ++++++---- 19 files changed, 80 insertions(+), 47 deletions(-) diff --git a/Compression.Core/DiskImage/PartitionedDiskLister.cs b/Compression.Core/DiskImage/PartitionedDiskLister.cs index 601965ba1..f64a1b4d9 100644 --- a/Compression.Core/DiskImage/PartitionedDiskLister.cs +++ b/Compression.Core/DiskImage/PartitionedDiskLister.cs @@ -175,7 +175,7 @@ private static string MakePartitionPrefix(PartitionEntry part) { // ── Add / Remove (partition-aware) ──────────────────────────────────── /// - /// Partition-aware . Returns true + /// Partition-aware . Returns true /// when a partition table was present and at least one input was dispatched /// through the partition-aware path; false when no partition table /// exists so the caller can fall through to the existing single-FS path. @@ -243,7 +243,7 @@ public static bool TryAdd(Stream disk, IReadOnlyList inputs) { } /// - /// Partition-aware . Returns true + /// Partition-aware . Returns true /// when a partition table was present and at least one entry was removed; /// false if no partition table was detected. Entry name shapes: /// Partition<N>_<Type>/<inner> deletes an inner-FS diff --git a/Compression.Registry/IArchiveWriteConstraints.cs b/Compression.Registry/IArchiveWriteConstraints.cs index 2a94d64cb..fe436c9bb 100644 --- a/Compression.Registry/IArchiveWriteConstraints.cs +++ b/Compression.Registry/IArchiveWriteConstraints.cs @@ -4,7 +4,7 @@ namespace Compression.Registry; /// Opt-in capability: the descriptor can reject inputs that don't belong in this archive /// type. Applied by the UI (drag-drop prohibition cursor + tooltip) and the CLI (rejection /// with non-zero exit) before or -/// is called. +/// is called. /// /// Descriptors that accept anything (ZIP, TAR, …) simply don't implement this interface. /// diff --git a/Compression.Registry/RebuildVerb.cs b/Compression.Registry/RebuildVerb.cs index 42b234bd9..3ee3feec6 100644 --- a/Compression.Registry/RebuildVerb.cs +++ b/Compression.Registry/RebuildVerb.cs @@ -482,7 +482,7 @@ private static string SafeExtractPath(string root, string archiveName) { } /// A writable scratch stream not bounded by byte[] / MemoryStream size. - internal static FileStream CreateScratchStream() + public static FileStream CreateScratchStream() => new(Path.Combine(Path.GetTempPath(), "cwb_rebuild_" + Guid.NewGuid().ToString("N") + ".tmp"), FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 64 * 1024, FileOptions.DeleteOnClose); diff --git a/Compression.Tests/Cso/CsoInPlaceModifyTests.cs b/Compression.Tests/Cso/CsoInPlaceModifyTests.cs index d9c16aa44..51b326132 100644 --- a/Compression.Tests/Cso/CsoInPlaceModifyTests.cs +++ b/Compression.Tests/Cso/CsoInPlaceModifyTests.cs @@ -204,7 +204,7 @@ private static byte[] ReadLogicalPayload(byte[] image) { using var result = new MemoryStream(); foreach (var name in names) { stream.Position = 0; - result.Write(descriptor.ExtractEntryToMemory(stream, name, null)); + result.Write(((IArchiveFormatOperations)descriptor).ExtractEntryToMemory(stream, name, null)); } return result.ToArray(); } diff --git a/Compression.Tests/Cso/CsoTests.cs b/Compression.Tests/Cso/CsoTests.cs index 5c0108fa4..60c83036f 100644 --- a/Compression.Tests/Cso/CsoTests.cs +++ b/Compression.Tests/Cso/CsoTests.cs @@ -40,7 +40,7 @@ public void Zso_DecodesLibLz4KnownBlock() { var descriptor = new CsoFormatDescriptor(); using var stream = new MemoryStream(image); - var block = descriptor.ExtractEntryToMemory(stream, "blocks/block_00000.bin", null); + var block = ((IArchiveFormatOperations)descriptor).ExtractEntryToMemory(stream, "blocks/block_00000.bin", null); Assert.That(block, Is.EqualTo(new byte[BlockSize])); stream.Position = 0; @@ -55,7 +55,7 @@ public void CsoV2_DecodesLibLz4KnownBlock() { var descriptor = new CsoFormatDescriptor(); using var stream = new MemoryStream(image); - var block = descriptor.ExtractEntryToMemory(stream, "blocks/block_00000.bin", null); + var block = ((IArchiveFormatOperations)descriptor).ExtractEntryToMemory(stream, "blocks/block_00000.bin", null); Assert.That(block, Is.EqualTo(new byte[BlockSize])); stream.Position = 0; @@ -149,7 +149,7 @@ private static byte[] ReadLogicalPayload(byte[] image) { using var output = new MemoryStream(); foreach (var name in names) { stream.Position = 0; - output.Write(descriptor.ExtractEntryToMemory(stream, name, null)); + output.Write(((IArchiveFormatOperations)descriptor).ExtractEntryToMemory(stream, name, null)); } return output.ToArray(); } diff --git a/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs b/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs index 5c978528c..ba7336808 100644 --- a/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs +++ b/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs @@ -44,7 +44,11 @@ public void CredentialAwareAdd_ReplacesPayload_AndPreservesCipherProfile() { var replacement = Enumerable.Range(0, 12001).Select(i => (byte)(i * 41 + 3)).ToArray(); var descriptor = new EcryptfsFormatDescriptor(); var modifier = (IArchiveModifiable)descriptor; - using var image = new MemoryStream(CreateImage([1, 2, 3, 4, 5], password, "aes256")); + // Expandable: MemoryStream(byte[]) is fixed-capacity, and the replacement payload here is + // larger than the original, so the Add has to grow the image. + using var image = new MemoryStream(); + var seed = CreateImage([1, 2, 3, 4, 5], password, "aes256"); + image.Write(seed, 0, seed.Length); image.Position = 0; string? signatureBefore; diff --git a/Compression.Tests/FirmwareHex/TiTxtMaintenanceTests.cs b/Compression.Tests/FirmwareHex/TiTxtMaintenanceTests.cs index b2c6bb8e0..50aa144d2 100644 --- a/Compression.Tests/FirmwareHex/TiTxtMaintenanceTests.cs +++ b/Compression.Tests/FirmwareHex/TiTxtMaintenanceTests.cs @@ -112,7 +112,15 @@ public void Create_RejectsMultiplePayloads() { Throws.InstanceOf()); } - private static MemoryStream StreamOf(string text) => new(Encoding.ASCII.GetBytes(text)); + // Built empty and written into: MemoryStream(byte[]) is fixed-capacity, and the maintenance + // verbs under test grow the image. + private static MemoryStream StreamOf(string text) { + var result = new MemoryStream(); + var bytes = Encoding.ASCII.GetBytes(text); + result.Write(bytes, 0, bytes.Length); + result.Position = 0; + return result; + } private static FirmwareImage Parse(MemoryStream stream) { stream.Position = 0; diff --git a/Compression.Tests/Lustre/LustreDetectionTests.cs b/Compression.Tests/Lustre/LustreDetectionTests.cs index 3f5d9975d..a2265df85 100644 --- a/Compression.Tests/Lustre/LustreDetectionTests.cs +++ b/Compression.Tests/Lustre/LustreDetectionTests.cs @@ -124,7 +124,7 @@ private static (int BlockSize, long BlockOffset, long DescriptorOffset) FindInit if ((featureIncompat & 0x80) != 0) totalBlocks |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(0x150, 4)) << 32; var descriptorSize = (featureIncompat & 0x80) != 0 - ? Math.Max(32, BinaryPrimitives.ReadUInt16LittleEndian(sb.Slice(0xFE, 2))) + ? Math.Max(32, (int)BinaryPrimitives.ReadUInt16LittleEndian(sb.Slice(0xFE, 2))) : 32; var groupCount = (totalBlocks - firstDataBlock + blocksPerGroup - 1) / blocksPerGroup; var descriptorTableOffset = (long)(firstDataBlock + 1UL) * blockSize; diff --git a/Compression.Tests/Nrg/NrgAuthoringTests.cs b/Compression.Tests/Nrg/NrgAuthoringTests.cs index 4ce750dd9..b54d76fad 100644 --- a/Compression.Tests/Nrg/NrgAuthoringTests.cs +++ b/Compression.Tests/Nrg/NrgAuthoringTests.cs @@ -53,15 +53,18 @@ public void Write_MixedModeDao_PreservesAudioAndFindsDataTrack() { var dataStart = checked((long)BinaryPrimitives.ReadUInt64BigEndian(dataRecord[26..])); var dataEnd = checked((long)BinaryPrimitives.ReadUInt64BigEndian(dataRecord[34..])); + // Materialised before the closure: a lambda cannot capture a ref struct. + var audioBytes = audioRecord.ToArray(); + var dataBytes = dataRecord.ToArray(); Assert.Multiple(() => { - Assert.That(BinaryPrimitives.ReadUInt16BigEndian(audioRecord[12..]), Is.EqualTo(AudioSectorSize)); - Assert.That(audioRecord[14], Is.EqualTo((byte)NrgTrackMode.Audio)); - Assert.That(System.Text.Encoding.ASCII.GetString(audioRecord[..12]), Is.EqualTo("USABC2600001")); + Assert.That(BinaryPrimitives.ReadUInt16BigEndian(audioBytes.AsSpan(12)), Is.EqualTo(AudioSectorSize)); + Assert.That(audioBytes[14], Is.EqualTo((byte)NrgTrackMode.Audio)); + Assert.That(System.Text.Encoding.ASCII.GetString(audioBytes.AsSpan(0, 12)), Is.EqualTo("USABC2600001")); Assert.That(audioPregap, Is.Zero); Assert.That(audioStart - audioPregap, Is.EqualTo(150L * AudioSectorSize)); Assert.That(audioEnd - audioStart, Is.EqualTo(audio.Length)); - Assert.That(BinaryPrimitives.ReadUInt16BigEndian(dataRecord[12..]), Is.EqualTo(CookedSectorSize)); - Assert.That(dataRecord[14], Is.EqualTo((byte)NrgTrackMode.Mode1)); + Assert.That(BinaryPrimitives.ReadUInt16BigEndian(dataBytes.AsSpan(12)), Is.EqualTo(CookedSectorSize)); + Assert.That(dataBytes[14], Is.EqualTo((byte)NrgTrackMode.Mode1)); Assert.That(dataStart, Is.EqualTo(audioEnd)); Assert.That(dataEnd - dataStart, Is.EqualTo(iso.Length)); }); diff --git a/Compression.Tests/Nrg/NrgTests.cs b/Compression.Tests/Nrg/NrgTests.cs index d4d75934a..7d9e568a7 100644 --- a/Compression.Tests/Nrg/NrgTests.cs +++ b/Compression.Tests/Nrg/NrgTests.cs @@ -111,41 +111,41 @@ public void Create_WritesRealV2DaoChunkChain() { Assert.That(trailerOffset % 2048, Is.Zero); var position = trailerOffset; - var cuex = ReadChunk(image, ref position, "CUEX", 32); + var cuex = ReadChunk(image, ref position, "CUEX", 32).ToArray(); Assert.Multiple(() => { Assert.That(cuex[0], Is.EqualTo(0x41)); Assert.That(cuex[1], Is.EqualTo(0x00)); Assert.That(cuex[2], Is.EqualTo(0x00)); - Assert.That(unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(cuex[4..])), Is.EqualTo(-150)); + Assert.That(unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(cuex.AsSpan(4))), Is.EqualTo(-150)); Assert.That(cuex[9], Is.EqualTo(0x01)); Assert.That(cuex[10], Is.EqualTo(0x00)); - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex[12..]), Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex.AsSpan(12)), Is.Zero); Assert.That(cuex[17], Is.EqualTo(0x01)); Assert.That(cuex[18], Is.EqualTo(0x01)); - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex[20..]), Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex.AsSpan(20)), Is.Zero); Assert.That(cuex[25], Is.EqualTo(0xAA)); Assert.That(cuex[26], Is.EqualTo(0x01)); - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex[28..]), Is.EqualTo((uint)(trailerOffset / 2048))); + Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex.AsSpan(28)), Is.EqualTo((uint)(trailerOffset / 2048))); }); - var daox = ReadChunk(image, ref position, "DAOX", 64); - var track = daox[22..]; + var daox = ReadChunk(image, ref position, "DAOX", 64).ToArray(); + var track = daox.AsSpan(22).ToArray(); Assert.Multiple(() => { Assert.That(BinaryPrimitives.ReadUInt32BigEndian(daox), Is.EqualTo(64)); Assert.That(daox[20], Is.EqualTo(1)); Assert.That(daox[21], Is.EqualTo(1)); - Assert.That(BinaryPrimitives.ReadUInt16BigEndian(track[12..]), Is.EqualTo(2048)); + Assert.That(BinaryPrimitives.ReadUInt16BigEndian(track.AsSpan(12)), Is.EqualTo(2048)); Assert.That(track[14], Is.Zero, "cooked Mode 1"); - Assert.That(BinaryPrimitives.ReadUInt16BigEndian(track[16..]), Is.EqualTo(1)); - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track[18..]), Is.Zero); - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track[26..]), Is.Zero); - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track[34..]), Is.EqualTo((ulong)trailerOffset)); + Assert.That(BinaryPrimitives.ReadUInt16BigEndian(track.AsSpan(16)), Is.EqualTo(1)); + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track.AsSpan(18)), Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track.AsSpan(26)), Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track.AsSpan(34)), Is.EqualTo((ulong)trailerOffset)); }); - var sinf = ReadChunk(image, ref position, "SINF", 4); + var sinf = ReadChunk(image, ref position, "SINF", 4).ToArray(); Assert.That(BinaryPrimitives.ReadUInt32BigEndian(sinf), Is.EqualTo(1)); - var mtyp = ReadChunk(image, ref position, "MTYP", 4); + var mtyp = ReadChunk(image, ref position, "MTYP", 4).ToArray(); Assert.That(BinaryPrimitives.ReadUInt32BigEndian(mtyp), Is.EqualTo(0x00000400)); _ = ReadChunk(image, ref position, "END!", 0); diff --git a/Compression.Tests/Refs/RefsBlockCloneExternalTests.cs b/Compression.Tests/Refs/RefsBlockCloneExternalTests.cs index cd74a9d17..650132238 100644 --- a/Compression.Tests/Refs/RefsBlockCloneExternalTests.cs +++ b/Compression.Tests/Refs/RefsBlockCloneExternalTests.cs @@ -1,6 +1,7 @@ #pragma warning disable CS1591 using System.ComponentModel; using System.Diagnostics; +using Category = NUnit.Framework.CategoryAttribute; using FileSystem.Refs; namespace Compression.Tests.Refs; diff --git a/Compression.Tests/Support/MaintenanceOperationProbe.cs b/Compression.Tests/Support/MaintenanceOperationProbe.cs index c421059a3..c36f738c2 100644 --- a/Compression.Tests/Support/MaintenanceOperationProbe.cs +++ b/Compression.Tests/Support/MaintenanceOperationProbe.cs @@ -59,8 +59,17 @@ public static ProbeImage CreateImage(string formatId, string workDirectory) { var image = Path.Combine(workDirectory, "probe.img"); try { - ArchiveOperations.Create(image, [new ArchiveInput(sourcePath, ProbeName)], - new CompressionOptions { Password = PasswordFor(ops!) }, format, null); + // Unencrypted by default. Defragment(Stream) and Purge take no credentials, so a probe + // encrypted merely because the format *supports* passwords is one those verbs cannot read -- + // which is every classic archive format. Only a format that refuses to be created without + // one gets the password, on the retry below. + try { + ArchiveOperations.Create(image, [new ArchiveInput(sourcePath, ProbeName)], + new CompressionOptions(), format, null); + } catch when (PasswordFor(ops!) != null) { + ArchiveOperations.Create(image, [new ArchiveInput(sourcePath, ProbeName)], + new CompressionOptions { Password = PasswordFor(ops!) }, format, null); + } } catch (Exception ex) { Assert.Fail($"{formatId}: advertises a maintenance verb but neither creates the standard probe image " + $"nor declares the input inadmissible through IArchiveWriteConstraints: {ex.GetType().Name}: {ex.Message}"); diff --git a/Compression.UI/Views/DefragmentWindow.xaml.cs b/Compression.UI/Views/DefragmentWindow.xaml.cs index 7b58754c4..ddfeb71a5 100644 --- a/Compression.UI/Views/DefragmentWindow.xaml.cs +++ b/Compression.UI/Views/DefragmentWindow.xaml.cs @@ -1687,7 +1687,7 @@ private void OnWipeEmpty(object sender, RoutedEventArgs e) { /// /// Runs the purge operation: erases all live data from the container, /// leaving a valid but empty archive/image. Realised via - /// over every entry. Distinct from + /// over every entry. Distinct from /// wipe-empty, which only zeros dead space (see docs/ARCHIVE-MODEL.md). /// private void OnPurge(object sender, RoutedEventArgs e) { diff --git a/FileFormats/FileFormat.Macrium/MacriumFormatDescriptor.cs b/FileFormats/FileFormat.Macrium/MacriumFormatDescriptor.cs index c6faf2b27..1d5a148d8 100644 --- a/FileFormats/FileFormat.Macrium/MacriumFormatDescriptor.cs +++ b/FileFormats/FileFormat.Macrium/MacriumFormatDescriptor.cs @@ -36,8 +36,8 @@ namespace FileFormat.Macrium; /// /// R/W semantics: Macrium Reflect X is a disk-image format whose /// logical payload is a single contiguous sector stream — the same shape as -/// VHD / VDI / VMDK / QCOW2. and -/// operate on the synthetic +/// VHD / VDI / VMDK / QCOW2. and +/// operate on the synthetic /// disk-image.raw entry (Add concatenates supplied input bytes onto /// the existing image and rebuilds the container; Add of an entry whose name /// matches an existing one replaces the disk payload; Remove of diff --git a/FileFormats/FileFormat.Nrg/NrgWriter.cs b/FileFormats/FileFormat.Nrg/NrgWriter.cs index 3af39d86a..2dcc48e86 100644 --- a/FileFormats/FileFormat.Nrg/NrgWriter.cs +++ b/FileFormats/FileFormat.Nrg/NrgWriter.cs @@ -107,11 +107,13 @@ public static void Write(Stream output, NrgDiscDefinition disc) { var sessions = WriteTrackData(output, disc); var trailerOffset = checked((ulong)output.Position); + // Allocated once, outside the loop: a stackalloc per iteration grows the frame with the + // session count (CA2014). The four bytes are fully rewritten each time. + Span sinf = stackalloc byte[4]; foreach (var session in sessions) { WriteCuex(output, session); WriteDaox(output, session); - Span sinf = stackalloc byte[4]; BinaryPrimitives.WriteUInt32BigEndian(sinf, checked((uint)session.Tracks.Count)); WriteChunk(output, "SINF"u8, sinf); } diff --git a/FileFormats/FileFormat.UefiFv/UefiFvReader.cs b/FileFormats/FileFormat.UefiFv/UefiFvReader.cs index ec09bd5fe..d0871be0e 100644 --- a/FileFormats/FileFormat.UefiFv/UefiFvReader.cs +++ b/FileFormats/FileFormat.UefiFv/UefiFvReader.cs @@ -45,9 +45,13 @@ public static FirmwareVolume Read(ReadOnlySpan data, int fvStart = 0) { } if (!terminated) throw new InvalidDataException("UefiFv: firmware-volume block map is not terminated inside HeaderLength."); - var files = UefiFvParser.LiveSlots(layout).Select(slot => new FfsFile( - slot.Name, slot.Type, slot.Attributes, slot.RawState, checked((uint)slot.Size), - data.Slice(slot.DataOffset, slot.DataLength).ToArray())).ToList(); + // `data` is a ReadOnlySpan, which a lambda cannot capture, so the slices are materialised + // here rather than inside the projection. + var files = new List(); + foreach (var slot in UefiFvParser.LiveSlots(layout)) + files.Add(new FfsFile( + slot.Name, slot.Type, slot.Attributes, slot.RawState, checked((uint)slot.Size), + data.Slice(slot.DataOffset, slot.DataLength).ToArray())); var header = new FvHeader(new Guid(data.Slice(fvStart + 16, 16)), fvLength, attributes, headerLength, checksum, extOff, revision, blockMap); return new FirmwareVolume(fvStart, header, files); diff --git a/FileSystems/FileSystem.SysV/SysVFormatDescriptor.cs b/FileSystems/FileSystem.SysV/SysVFormatDescriptor.cs index c9f292655..06a2b4032 100644 --- a/FileSystems/FileSystem.SysV/SysVFormatDescriptor.cs +++ b/FileSystems/FileSystem.SysV/SysVFormatDescriptor.cs @@ -211,7 +211,7 @@ public void Add(Stream archive, IReadOnlyList inputs) { /// /// Removes the named entries from an existing s5fs image via the in-place /// modifier (zeroes file data blocks before returning them to the free - /// list, matching the wipe + /// list, matching the wipe /// contract). Falls back to the rebuild path for any nested-path entry /// the in-place engine won't touch. /// diff --git a/FileSystems/FileSystem.SysV/SysVInPlaceModifier.cs b/FileSystems/FileSystem.SysV/SysVInPlaceModifier.cs index e21a87425..2c4ac71d4 100644 --- a/FileSystems/FileSystem.SysV/SysVInPlaceModifier.cs +++ b/FileSystems/FileSystem.SysV/SysVInPlaceModifier.cs @@ -29,7 +29,7 @@ namespace FileSystem.SysV; /// data block stays byte-identical at its original offset. /// Remove: zero the dirent's inode number in place; wipe each /// data block the inode addresses to all-zero (matching the -/// wipe contract); push the data +/// wipe contract); push the data /// blocks onto s_free[] (spilling the cache to a new chain group /// when the 50th entry would overflow); zero the 64-byte inode slot so /// the re-scan rediscovers it. Every untouched inode and every untouched diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index cf4b6719f..b9f7f63b0 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -68,7 +68,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [MDF/MDS](https://en.wikipedia.org/wiki/Alcohol_120%25) | `Mdf` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Alcohol 120%](https://en.wikipedia.org/wiki/Alcohol_120%25) | | [Nintendo PartitionFS](https://en.wikipedia.org/wiki/Nintendo_Switch) | `Pfs0` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | Nintendo Switch PartitionFS | [PFS0 (switchbrew)](https://switchbrew.org/wiki/NCA#PFS0) | | [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | ✅ | ✅ rebuild | — | ✅ | — | ✅ | own reader + struct-parity tests | NRG v1/v2 reader; v2 TAO writer; named edits rebuild the embedded ISO | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | -| [PSP CSO/ZSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | `Cso` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Compressed ISO; edits re-pack the affected blocks | [CSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | +| [PSP CSO/ZSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | `Cso` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Compressed ISO; edits re-pack the affected blocks | [CSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | | [QCOW2](https://en.wikipedia.org/wiki/Qcow) | `Qcow2` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | [`qemu-img check`](https://www.qemu.org/docs/master/tools/qemu-img.html) / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [QCOW2 specification](https://www.qemu.org/docs/master/interop/qcow2.html) | | [T64](https://en.wikipedia.org/wiki/Commodore_64) | `T64` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 tape archive | [T64 (VICE)](https://vice-emu.sourceforge.io/vice_17.html) | | [TAP](https://en.wikipedia.org/wiki/ZX_Spectrum) | `Tap` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | Sinclair / Commodore tape image | [TAP (World of Spectrum)](https://worldofspectrum.net/faq/reference/formats.htm) | @@ -97,6 +97,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [ReFS](https://en.wikipedia.org/wiki/ReFS) | `Refs` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | ReFS 3.x; mutation is offline-quiescent only (see limitations). Its geometry analysis is real, but there is no creator to write a re-laid volume with, so Layout stays absent | [ReFS overview](https://learn.microsoft.com/windows-server/storage/refs/refs-overview) | | [Stacker CVF](https://en.wikipedia.org/wiki/Stac_Electronics#Stacker) | `Stacker` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | `dmsdos` mounts the genuine `GenuineStackerWriter` output and reads byte-exact | STACVOL; Genuine/Extended flavor-preserving rebuild edits | [Stacker notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/FileSystems/FileSystem.Stacker/FORMAT-NOTES.md) | | [Transactional FAT (TFAT)](https://learn.microsoft.com/previous-versions/windows/embedded/aa911939(v=msdn.10)) | `TFat` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount + [`fsck.fat`](https://github.com/dosfstools/dosfstools) read the written volume | Transaction-safe FAT (Windows CE) | [TFAT](https://learn.microsoft.com/previous-versions/windows/embedded/aa911939(v=msdn.10)) | + ### Unix / Linux | Format | Id | State | Compact | Defrag | Wipe | Shrink | Layout | Purge | Proof | Notes | Reference | @@ -128,7 +129,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Reiser4](https://en.wikipedia.org/wiki/Reiser4) | `Reiser4` | WORM | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; `reiser4progs` not installable here | Empty tree from `mkfs.reiser4` captures; files live in a payload area the reiser4 driver does not see, so the writer stays WORM | [Reiser4](https://reiser4.wiki.kernel.org/) | | [ReiserFS](https://en.wikipedia.org/wiki/ReiserFS) | `ReiserFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`reiserfsck`](https://git.kernel.org/pub/scm/linux/kernel/git/jeffm/reiserfsprogs.git), also after mutation | ReiserFS 3.6; edits rebuild the tree | [ReiserFS documentation](https://www.kernel.org/doc/html/latest/filesystems/reiserfs.html) | | [TUX2](https://en.wikipedia.org/wiki/Tux2) | `Tux2` | R | — | — | — | — | — | — | own reader + struct-parity tests | Phase-tree filesystem | [Tux2](https://en.wikipedia.org/wiki/Tux2) | -| [TUX3](https://en.wikipedia.org/wiki/Tux3) | `Tux3` | R | — | — | — | — | — | — | own reader + struct-parity tests | — | [Tux3](https://github.com/OGAWAHirofumi/tux3) | +| [TUX3](https://en.wikipedia.org/wiki/Tux3) | `Tux3` | R | ✅ | — | ✅ | ✅ | — | — | own reader + struct-parity tests | — | [Tux3](https://github.com/OGAWAHirofumi/tux3) | | [UFS](https://en.wikipedia.org/wiki/Unix_File_System) | `Ufs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | FreeBSD kernel mount r/w + [`fsck_ffs`](https://man.freebsd.org/cgi/man.cgi?query=fsck_ffs) under QEMU | UFS1 / FFS | [UFS](https://en.wikipedia.org/wiki/Unix_File_System) | | [UNIX System V FS](https://en.wikipedia.org/wiki/Unix_File_System) | `SysV` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `sysv` mount reads byte-exact | System V filesystem | [sysv](https://www.kernel.org/doc/html/latest/filesystems/sysv-fs.html) | | [VxFS (Veritas)](https://en.wikipedia.org/wiki/Veritas_File_System) | `VxFs` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | Linux `freevxfs` driver mounts the written volume | One fileset, direct extents, flat root; edits rebuild the volume | [freevxfs](https://github.com/torvalds/linux/tree/master/fs/freevxfs) | @@ -171,6 +172,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [YAFFS2](https://en.wikipedia.org/wiki/YAFFS) | `Yaffs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | — | [YAFFS](https://yaffs.net/) | ### Amiga, Atari, Acorn and other home computers + | Format | Id | State | Compact | Defrag | Wipe | Shrink | Layout | Purge | Proof | Notes | Reference | | --- | --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | --- | --- | --- | | [Acorn ADFS](https://en.wikipedia.org/wiki/Advanced_Disc_Filing_System) | `Adfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; the `adfs` module is absent from the QEMU guest | Acorn ADFS | [ADFS](https://en.wikipedia.org/wiki/Advanced_Disc_Filing_System) | @@ -216,16 +218,16 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [BeeGFS](https://en.wikipedia.org/wiki/BeeGFS) | `BeeGfs` | R | — | — | — | — | — | — | detection of the on-disk signature | Server-side objects only; no self-contained image exists | [BeeGFS](https://www.beegfs.io/) | | [CephFS / RADOS](https://en.wikipedia.org/wiki/Ceph_(software)) | `CephFs` | R | — | — | — | — | — | — | detection of the on-disk signature | RADOS objects only | [Ceph](https://docs.ceph.com/) | | [Dell EMC Isilon OneFS](https://en.wikipedia.org/wiki/OneFS_distributed_file_system) | `OneFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Isilon OneFS | [OneFS](https://www.dell.com/en-us/dt/storage/powerscale.htm) | -| [eCryptfs](https://en.wikipedia.org/wiki/ECryptfs) | `Ecryptfs` | WORM | ✅ | — | ✅ | ✅ | — | ✅ | [ecryptfs-utils passphrase vector](https://github.com/dustinkirkland/ecryptfs-utils/blob/master/tests/userspace/verify-passphrase-sig.sh) + Linux kernel layout | Passphrase lower-file AES-128/192/256 read/create; private-key auth and xattr-only metadata are unsupported | [eCryptfs](https://www.kernel.org/doc/html/latest/filesystems/ecryptfs.html) | -| [GlusterFS](https://en.wikipedia.org/wiki/Gluster) | `GlusterFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Brick metadata only | [GlusterFS](https://docs.gluster.org/) | +| [eCryptfs](https://en.wikipedia.org/wiki/ECryptfs) | `Ecryptfs` | R/W | ✅ | — | ✅ | ✅ | — | ✅ | [ecryptfs-utils passphrase vector](https://github.com/dustinkirkland/ecryptfs-utils/blob/master/tests/userspace/verify-passphrase-sig.sh) + Linux kernel layout | Passphrase lower-file AES-128/192/256 read/create; private-key auth and xattr-only metadata are unsupported | [eCryptfs](https://www.kernel.org/doc/html/latest/filesystems/ecryptfs.html) | +| [GlusterFS brick](https://en.wikipedia.org/wiki/Gluster) | `GlusterFs` | R | ✅ | — | — | ✅ | — | — | detection of the on-disk signature | Brick metadata only | [GlusterFS](https://docs.gluster.org/) | | [IBM Spectrum Scale / GPFS](https://en.wikipedia.org/wiki/GPFS) | `Gpfs` | R | — | — | — | — | — | — | detection of the on-disk signature | IBM Spectrum Scale | [GPFS](https://www.ibm.com/docs/en/storage-scale) | | [JuiceFS](https://juicefs.com/) | `JuiceFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Chunk objects only | [JuiceFS](https://juicefs.com/docs/community/introduction/) | | [Lustre](https://en.wikipedia.org/wiki/Lustre_(file_system)) | `Lustre` | R | ✅ | — | ✅ | ✅ | — | — | own reader + struct-parity tests | ldiskfs MDT/OST backing store; conservative offline free-block wipe + trailing-block shrink; namespace edits remain R/O | [Lustre](https://www.lustre.org/) | -| [MooseFS](https://en.wikipedia.org/wiki/Moose_File_System) | `MooseFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Chunk-server objects only | [MooseFS](https://moosefs.com/) | +| [MooseFS](https://en.wikipedia.org/wiki/Moose_File_System) | `MooseFs` | R | — | — | ✅ | — | — | ✅ | detection of the on-disk signature | Chunk-server objects only | [MooseFS](https://moosefs.com/) | | [NetApp WAFL](https://en.wikipedia.org/wiki/Write_Anywhere_File_Layout) | `Wafl` | R | — | — | — | — | — | — | detection of the on-disk signature | NetApp WAFL | [WAFL](https://en.wikipedia.org/wiki/Write_Anywhere_File_Layout) | | [NSS (Novell Storage Services)](https://en.wikipedia.org/wiki/Novell_Storage_Services) | `Nss` | WORM | ✅ | ✅ moving | ✅ | — | — | — | anchor detection derived from real OES media; see the on-disk notes | Beast / B-tree layout undocumented; the writer emits a container under its own magic, not a pool | [NSS on-disk notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/NSS-ON-DISK.md) | | [NWFS (Novell NetWare 386 Traditional Filesystem)](https://en.wikipedia.org/wiki/NetWare_File_System) | `Nwfs` | R | — | — | — | — | — | — | detection of the on-disk signature | NetWare 286 / 3.x | [NWFS](https://en.wikipedia.org/wiki/NetWare_File_System) | -| [NWFS386 (Novell NetWare 386 Traditional Filesystem)](https://en.wikipedia.org/wiki/NetWare_File_System) | `Nwfs386` | R/W | ✅ | ✅ rebuild | — | ✅ | ✅ | ✅ | own reader + struct-parity tests | Plain single-segment DOS namespace; rebuild edits preserve block size and volume name; compression/suballocation/alternate namespaces are read-only | [Traditional NetWare partitions](https://www.novell.com/developer/documentation/vfs/vfs__enu/data/afpro44.html) | +| [NWFS386 (Novell Traditional NetWare filesystem)](https://en.wikipedia.org/wiki/NetWare_File_System) | `Nwfs386` | R/W | ✅ | ✅ rebuild | — | ✅ | ✅ | ✅ | own reader + struct-parity tests | Plain single-segment DOS namespace; rebuild edits preserve block size and volume name; compression/suballocation/alternate namespaces are read-only | [Traditional NetWare partitions](https://www.novell.com/developer/documentation/vfs/vfs__enu/data/afpro44.html) | | [OrangeFS / PVFS2 DBPF](https://en.wikipedia.org/wiki/OrangeFS) | `OrangeFs` | R/W | ✅ | ✅ rebuild | — | — | — | ✅ | own reader + struct-parity tests | One DBPF storage object, not a namespace; the opaque payload is what is edited | [OrangeFS](https://github.com/waltligon/orangefs) | | [SGI CXFS (Cluster XFS)](https://en.wikipedia.org/wiki/CXFS) | `Cxfs` | R | — | — | — | — | — | — | detection of the on-disk signature | SGI cluster XFS | [CXFS](https://en.wikipedia.org/wiki/CXFS) | | [Tahoe-LAFS share](https://en.wikipedia.org/wiki/Tahoe-LAFS) | `TahoeLafs` | R | — | — | — | — | — | — | detection of the on-disk signature | Shares are capability-encrypted; no read-cap, no plaintext | [Tahoe-LAFS](https://tahoe-lafs.org/) | From ec5eaf3e7f0104e1bb16d5958eb101ddcd6d8f88 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 13:20:47 +0200 Subject: [PATCH 162/165] Revert "Merge remote-tracking branch 'origin/fix/t64-rw-maintenance' into integrate/cw-round1b" This reverts commit b9aefe57c6c745cc19ef69c485634cf0ca58c5b1, reversing changes made to 1e7aaa357c67780701fd01f35388e9abfa2b63c1. --- Compression.Tests/T64/T64MaintenanceTests.cs | 199 ------------- FileFormats/FileFormat.T64/T64BlockMover.cs | 119 +++----- FileFormats/FileFormat.T64/T64Entry.cs | 10 +- .../FileFormat.T64/T64FormatDescriptor.cs | 229 ++++----------- .../FileFormat.T64/T64InPlaceModifier.cs | 265 +++++++++--------- FileFormats/FileFormat.T64/T64Modifier.cs | 123 +++++++- FileFormats/FileFormat.T64/T64Reader.cs | 135 ++------- FileFormats/FileFormat.T64/T64Writer.cs | 66 ++--- 8 files changed, 397 insertions(+), 749 deletions(-) delete mode 100644 Compression.Tests/T64/T64MaintenanceTests.cs diff --git a/Compression.Tests/T64/T64MaintenanceTests.cs b/Compression.Tests/T64/T64MaintenanceTests.cs deleted file mode 100644 index 800ea02eb..000000000 --- a/Compression.Tests/T64/T64MaintenanceTests.cs +++ /dev/null @@ -1,199 +0,0 @@ -using System.Buffers.Binary; -using Compression.Registry; -using FileFormat.T64; - -namespace Compression.Tests.T64; - -[TestFixture] -public class T64MaintenanceTests { - - [Test, Category("Compatibility")] - public void Reader_Conv64BrokenEndAddress_UsesPhysicalPayloadExtent() { - var payload = "HELLO"u8.ToArray(); - var writer = new T64Writer(); - writer.AddFile("BROKEN", 0x0801, payload); - var image = writer.Build(); - - BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(64 + 4), 0xC3C6); - - using var stream = new MemoryStream(image); - using var reader = new T64Reader(stream); - Assert.That(reader.Entries, Has.Count.EqualTo(1)); - Assert.That(reader.Entries[0].Size, Is.EqualTo(payload.Length)); - Assert.That(reader.Extract(reader.Entries[0]), Is.EqualTo(payload)); - } - - [Test, Category("EdgeCase")] - public void Writer_EntryEndingAt10000_RoundTripsFullPayload() { - var payload = Enumerable.Range(0, 0x4000).Select(static i => (byte)i).ToArray(); - var writer = new T64Writer(); - writer.AddFile("TOPMEM", 0xC000, payload); - - using var stream = new MemoryStream(writer.Build()); - using var reader = new T64Reader(stream); - var entry = reader.Entries.Single(); - - Assert.That(entry.EndAddress, Is.Zero, "exclusive $10000 end must wrap in the 16-bit field"); - Assert.That(entry.Size, Is.EqualTo(payload.Length)); - Assert.That(reader.Extract(entry), Is.EqualTo(payload)); - } - - [Test, Category("ErrorHandling")] - public void Reader_DirectoryPastEof_Throws() { - var image = new byte[64]; - "C64S tape image file"u8.CopyTo(image); - BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(32), 0x0100); - BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(34), 1); - - using var stream = new MemoryStream(image); - Assert.Throws(() => _ = new T64Reader(stream)); - } - - [Test, Category("ErrorHandling")] - public void Add_InvalidReplacement_DoesNotDeleteExistingEntry() { - var oldPayload = "keep-me"u8.ToArray(); - var writer = new T64Writer(); - writer.AddFile("TARGET", oldPayload); - using var stream = Writable(writer.Build()); - - Assert.Throws(() => - T64InPlaceModifier.AddFile(stream, "TARGET", new byte[0x10000], 0x0801)); - - using var reader = new T64Reader(stream); - Assert.That(reader.Entries.Select(static e => e.Name), Is.EqualTo(new[] { "TARGET" })); - Assert.That(reader.Extract(reader.Entries.Single()), Is.EqualTo(oldPayload)); - } - - [Test, Category("RoundTrip")] - public void BlockMover_ForwardOverlap_IsMemmoveSafeAndDoesNotZeroDestination() { - using var stream = Writable([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); - - new T64BlockMover().MoveExtent(stream, 0, 2, 6, zeroSource: true); - - Assert.That(stream.ToArray(), Is.EqualTo(new byte[] { 0, 0, 1, 2, 3, 4, 5, 6, 9, 10 })); - } - - [Test, Category("RoundTrip")] - public void Defragment_MovesPayloadToDirectoryEnd_AndPreservesMetadata() { - var (stream, payload) = BuildGappedImage(); - using (stream) { - var descriptor = new T64FormatDescriptor(); - descriptor.Defragment(stream); - - Assert.That(stream.Length, Is.EqualTo(64 + 2 * 32 + payload.Length)); - using var reader = new T64Reader(stream); - var entry = reader.Entries.Single(); - Assert.Multiple(() => { - Assert.That(entry.DataOffset, Is.EqualTo(128)); - Assert.That(entry.StartAddress, Is.EqualTo(0x2000)); - Assert.That(entry.FileType, Is.EqualTo(0x81)); - Assert.That(reader.Version, Is.EqualTo(0x0101)); - Assert.That(reader.TapeName, Is.EqualTo("GAPPED TAPE")); - Assert.That(reader.Extract(entry), Is.EqualTo(payload)); - }); - } - } - - [Test, Category("RoundTrip")] - public void WipeUnusedSpace_ZerosFreeDirectorySlotAndPayloadGaps() { - var (stream, payload) = BuildGappedImage(); - using (stream) { - var descriptor = new T64FormatDescriptor(); - var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(stream); - var raw = stream.ToArray(); - - Assert.Multiple(() => { - Assert.That(wiped, Is.EqualTo(32 + 4 + 3)); - Assert.That(raw.AsSpan(96, 32).ToArray(), Is.All.EqualTo(0)); - Assert.That(raw.AsSpan(128, 4).ToArray(), Is.All.EqualTo(0)); - Assert.That(raw.AsSpan(132, payload.Length).ToArray(), Is.EqualTo(payload)); - Assert.That(raw.AsSpan(132 + payload.Length, 3).ToArray(), Is.All.EqualTo(0)); - }); - } - } - - [Test, Category("RoundTrip")] - public void Purge_LeavesValidCanonicalEmptyImage() { - var (stream, _) = BuildGappedImage(); - using (stream) { - var descriptor = new T64FormatDescriptor(); - ((IArchivePurgeable)descriptor).Purge(stream); - - Assert.That(stream.Length, Is.EqualTo(64)); - using var reader = new T64Reader(stream); - Assert.Multiple(() => { - Assert.That(reader.Entries, Is.Empty); - Assert.That(reader.DirectoryEntryCount, Is.Zero); - Assert.That(reader.UsedEntryCount, Is.Zero); - Assert.That(reader.Version, Is.EqualTo(0x0101)); - Assert.That(reader.TapeName, Is.EqualTo("GAPPED TAPE")); - }); - } - } - - [Test, Category("ErrorHandling")] - public void LayoutMap_UnknownRecordKind_FailsClosed() { - var (stream, _) = BuildGappedImage(); - using (stream) { - var raw = stream.ToArray(); - raw[64] = 2; - using var unknown = Writable(raw); - - var descriptor = new T64FormatDescriptor(); - Assert.That(descriptor.EnumerateLayout(unknown), Is.Empty, - "unknown T64 record kinds must never turn undecoded bytes into wipeable free space"); - } - } - - [Test, Category("HappyPath")] - public void Descriptor_ExposesOnlyFormatWideMaintenanceCapabilities() { - var descriptor = new T64FormatDescriptor(); - Assert.Multiple(() => { - Assert.That(descriptor, Is.InstanceOf()); - Assert.That(descriptor, Is.InstanceOf()); - Assert.That(descriptor, Is.InstanceOf()); - Assert.That(descriptor, Is.InstanceOf()); - Assert.That(descriptor, Is.Not.InstanceOf(), - "normal T64 records can be rebuilt tightly, but other defined record kinds cannot yet be faithfully re-authored"); - Assert.That(descriptor, Is.Not.InstanceOf(), - "T64 has no selectable filesystem/block geometry to re-layout."); - }); - } - - private static (MemoryStream Stream, byte[] Payload) BuildGappedImage() { - var payload = "PAYLOAD"u8.ToArray(); - const int payloadOffset = 132; // 64-byte header + 2 slots + 4 dead bytes - var image = new byte[payloadOffset + 7 + 3]; - - "C64S tape image file"u8.CopyTo(image); - BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(32), 0x0101); - BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(34), 2); - BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(36), 1); - "GAPPED TAPE"u8.CopyTo(image.AsSpan(40)); - image.AsSpan(51, 13).Fill(0x20); - - image[64] = 1; - image[65] = 0x81; - BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(66), 0x2000); - BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(68), (ushort)(0x2000 + payload.Length)); - BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(72), payloadOffset); - "GAPTEST"u8.CopyTo(image.AsSpan(80)); - image.AsSpan(87, 9).Fill(0x20); - - // Free directory slot containing stale metadata, then dead payload gap/tail. - image.AsSpan(96, 32).Fill(0xA5); - image[96] = 0; - image.AsSpan(128, 4).Fill(0xD1); - payload.CopyTo(image, payloadOffset); - image.AsSpan(payloadOffset + payload.Length, 3).Fill(0xD2); - - return (Writable(image), payload); - } - - private static MemoryStream Writable(byte[] bytes) { - var result = new MemoryStream(); - result.Write(bytes); - result.Position = 0; - return result; - } -} diff --git a/FileFormats/FileFormat.T64/T64BlockMover.cs b/FileFormats/FileFormat.T64/T64BlockMover.cs index bc80375bf..bb8157cb9 100644 --- a/FileFormats/FileFormat.T64/T64BlockMover.cs +++ b/FileFormats/FileFormat.T64/T64BlockMover.cs @@ -17,44 +17,34 @@ public sealed class T64BlockMover : IFilesystemBlockMover { /// public void MoveExtent(Stream image, long srcOffset, long dstOffset, long length, bool zeroSource = false) { - ArgumentNullException.ThrowIfNull(image); - if (!image.CanRead || !image.CanWrite || !image.CanSeek) - throw new ArgumentException("T64: block moves require a readable, writable, seekable stream.", nameof(image)); - if (srcOffset < 0) throw new ArgumentOutOfRangeException(nameof(srcOffset)); - if (dstOffset < 0) throw new ArgumentOutOfRangeException(nameof(dstOffset)); - if (length < 0) throw new ArgumentOutOfRangeException(nameof(length)); - if (length == 0 || srcOffset == dstOffset) return; - - var srcEnd = checked(srcOffset + length); - var dstEnd = checked(dstOffset + length); - if (srcEnd > image.Length) - throw new ArgumentOutOfRangeException(nameof(length), "T64: source extent extends beyond EOF."); - if (dstEnd > image.Length) - image.SetLength(dstEnd); - + if (length <= 0 || srcOffset == dstOffset) return; var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); try { - // memmove semantics: a forward-overlapping destination must be copied - // high-to-low or the first write destroys bytes we have not read yet. - if (dstOffset > srcOffset && dstOffset < srcEnd) - CopyBackward(image, srcOffset, dstOffset, length, buffer); - else - CopyForward(image, srcOffset, dstOffset, length, buffer); - - if (!zeroSource) return; - - Array.Clear(buffer, 0, buffer.Length); - var overlapStart = Math.Max(srcOffset, dstOffset); - var overlapEnd = Math.Min(srcEnd, dstEnd); - if (overlapStart >= overlapEnd) { - ZeroRange(image, srcOffset, length, buffer); - return; + var remaining = length; + var src = srcOffset; + var dst = dstOffset; + while (remaining > 0) { + var chunk = (int)Math.Min(remaining, buffer.Length); + image.Position = src; + image.ReadExactly(buffer, 0, chunk); + image.Position = dst; + image.Write(buffer, 0, chunk); + src += chunk; + dst += chunk; + remaining -= chunk; + } + if (zeroSource) { + Array.Clear(buffer, 0, buffer.Length); + remaining = length; + src = srcOffset; + while (remaining > 0) { + var chunk = (int)Math.Min(remaining, buffer.Length); + image.Position = src; + image.Write(buffer, 0, chunk); + src += chunk; + remaining -= chunk; + } } - - // Never zero the intersection: those bytes are now part of the moved - // destination. Scrub only the portions exclusively owned by the old run. - ZeroRange(image, srcOffset, overlapStart - srcOffset, buffer); - ZeroRange(image, overlapEnd, srcEnd - overlapEnd, buffer); } finally { ArrayPool.Shared.Return(buffer); } @@ -62,78 +52,33 @@ public void MoveExtent(Stream image, long srcOffset, long dstOffset, long length /// public void UpdateAllocationAfterMove(Stream image, string fileName, long oldOffset, long newOffset, long length) { - ArgumentNullException.ThrowIfNull(image); - ArgumentNullException.ThrowIfNull(fileName); - if (!image.CanRead || !image.CanWrite || !image.CanSeek) - throw new ArgumentException("T64: allocation updates require a readable, writable, seekable stream.", nameof(image)); - if (oldOffset < 0 || oldOffset > uint.MaxValue) throw new ArgumentOutOfRangeException(nameof(oldOffset)); - if (newOffset < 0 || newOffset > uint.MaxValue) throw new ArgumentOutOfRangeException(nameof(newOffset)); - image.Position = 34; Span hdrBuf = stackalloc byte[2]; image.ReadExactly(hdrBuf); var maxEntries = BinaryPrimitives.ReadUInt16LittleEndian(hdrBuf); - if ((long)HeaderSize + maxEntries * EntrySize > image.Length) - throw new InvalidDataException("T64: directory extends beyond end of image."); - Span entry = stackalloc byte[EntrySize]; for (var i = 0; i < maxEntries; i++) { var slotOff = HeaderSize + i * EntrySize; + var entry = new byte[EntrySize]; image.Position = slotOff; image.ReadExactly(entry); if (entry[0] == 0) continue; - var dataOffset = BinaryPrimitives.ReadUInt32LittleEndian(entry[8..]); + var dataOffset = BinaryPrimitives.ReadUInt32LittleEndian(entry.AsSpan(8)); if (dataOffset != (uint)oldOffset) continue; + // Optionally match by name. if (!string.Equals(fileName, "*", StringComparison.Ordinal)) { - var entryName = Encoding.ASCII.GetString(entry[16..32]).TrimEnd('\0', ' '); + var entryName = Encoding.ASCII.GetString(entry, 16, 16).TrimEnd('\0', ' '); if (!entryName.Equals(fileName, StringComparison.OrdinalIgnoreCase)) continue; } - BinaryPrimitives.WriteUInt32LittleEndian(entry[8..], (uint)newOffset); + // Patch data offset. + BinaryPrimitives.WriteUInt32LittleEndian(entry.AsSpan(8), (uint)newOffset); image.Position = slotOff; image.Write(entry); - return; - } - - throw new InvalidDataException($"T64: no directory entry points at payload offset {oldOffset}."); - } - - private static void CopyForward(Stream image, long src, long dst, long length, byte[] buffer) { - var cursor = 0L; - while (cursor < length) { - var chunk = (int)Math.Min(length - cursor, buffer.Length); - image.Position = src + cursor; - image.ReadExactly(buffer, 0, chunk); - image.Position = dst + cursor; - image.Write(buffer, 0, chunk); - cursor += chunk; - } - } - - private static void CopyBackward(Stream image, long src, long dst, long length, byte[] buffer) { - var remaining = length; - while (remaining > 0) { - var chunk = (int)Math.Min(remaining, buffer.Length); - var offset = remaining - chunk; - image.Position = src + offset; - image.ReadExactly(buffer, 0, chunk); - image.Position = dst + offset; - image.Write(buffer, 0, chunk); - remaining = offset; - } - } - - private static void ZeroRange(Stream image, long offset, long length, byte[] zeroBuffer) { - var remaining = length; - while (remaining > 0) { - var chunk = (int)Math.Min(remaining, zeroBuffer.Length); - image.Position = offset; - image.Write(zeroBuffer, 0, chunk); - offset += chunk; - remaining -= chunk; + break; } } } diff --git a/FileFormats/FileFormat.T64/T64Entry.cs b/FileFormats/FileFormat.T64/T64Entry.cs index ffca02331..8de898239 100644 --- a/FileFormats/FileFormat.T64/T64Entry.cs +++ b/FileFormats/FileFormat.T64/T64Entry.cs @@ -2,13 +2,9 @@ namespace FileFormat.T64; /// -/// Represents a T64 directory entry. +/// Represents a t 64 entry. /// public sealed class T64Entry { - /// - /// Gets the zero-based directory slot index. - /// - public int DirectoryIndex { get; init; } /// /// Gets or sets the name. /// @@ -26,10 +22,6 @@ public sealed class T64Entry { /// public byte EntryType { get; init; } // 1=normal, 3=snapshot /// - /// Gets the Commodore file type byte. - /// - public byte FileType { get; init; } - /// /// Gets or sets the start address. /// public ushort StartAddress { get; init; } diff --git a/FileFormats/FileFormat.T64/T64FormatDescriptor.cs b/FileFormats/FileFormat.T64/T64FormatDescriptor.cs index f68837eeb..cc63f3348 100644 --- a/FileFormats/FileFormat.T64/T64FormatDescriptor.cs +++ b/FileFormats/FileFormat.T64/T64FormatDescriptor.cs @@ -1,5 +1,4 @@ #pragma warning disable CS1591 -using System.Buffers; using Compression.Registry; using static Compression.Registry.FormatHelpers; @@ -11,15 +10,10 @@ namespace FileFormat.T64; /// References: /// /// Peter Schepers, "C64 File Formats: T64" — the classic reference document -/// https://vice-emu.sourceforge.io/ — VICE emulator and T64 documentation +/// https://vice-emu.sourceforge.io/ — VICE emulator — reference implementation reading/writing T64 /// /// -public sealed class T64FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, - IArchiveModifiable, IArchiveDefragmentable, IArchivePurgeable, IArchiveLayoutMap, IFilesystemBlockMover { - - private const int HeaderSize = 64; - private const int EntrySize = 32; - +public sealed class T64FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable, IArchiveDefragmentable, IArchiveLayoutMap, IFilesystemBlockMover { /// /// Gets the id. /// @@ -76,8 +70,8 @@ public sealed class T64FormatDescriptor : IFormatDescriptor, IArchiveFormatOpera /// Lists the entries in the supplied container. /// public List List(Stream stream, string? password) { - using var reader = new T64Reader(stream); - return reader.Entries.Select((e, i) => new ArchiveEntryInfo( + var r = new T64Reader(stream); + return r.Entries.Select((e, i) => new ArchiveEntryInfo( i, e.Name, e.Size, e.Size, "Stored", false, false, null )).ToList(); } @@ -86,10 +80,10 @@ public List List(Stream stream, string? password) { /// Decodes the supplied input. /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - using var reader = new T64Reader(stream); - foreach (var entry in reader.Entries) { - if (files != null && !MatchesFilter(entry.Name, files)) continue; - WriteFile(outputDir, entry.Name, reader.Extract(entry)); + var r = new T64Reader(stream); + foreach (var e in r.Entries) { + if (files != null && !MatchesFilter(e.Name, files)) continue; + WriteFile(outputDir, e.Name, r.Extract(e)); } } @@ -97,16 +91,21 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? /// Performs the create operation. /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { - var writer = new T64Writer(); + var w = new T64Writer(); foreach (var (name, data) in FlatFiles(inputs)) - writer.AddFile(name.Length > 16 ? name[..16] : name, data); - output.Write(writer.Build()); + w.AddFile(name.Length > 16 ? name[..16] : name, data); + output.Write(w.Build()); } // ── IArchiveModifiable (in-place) ───────────────────────────────────── /// - /// Adds or replaces files inside an existing T64 image without rebuilding it. + /// Adds (or replaces by name) files inside an existing T64 tape image via + /// . If a directory slot is free the entry + /// drops in directly and the new payload is appended at EOF. If the + /// directory is full the directory grows by one 32-byte slot — the payload + /// region shifts forward by 32 bytes and every existing slot's absolute + /// dataOffset field is patched. No full image rebuild. /// public void Add(Stream archive, IReadOnlyList inputs) { ArgumentNullException.ThrowIfNull(archive); @@ -116,7 +115,11 @@ public void Add(Stream archive, IReadOnlyList inputs) { } /// - /// Removes named entries, wipes their payload bytes and compacts the image. + /// Removes named entries from an existing T64 tape image via + /// . Later directory slots shift up by 32 + /// bytes, the removed payload bytes are wiped, the remaining payload region + /// shifts to close the gap (each affected slot's absolute dataOffset is + /// patched), and the stream is truncated. /// public void Remove(Stream archive, string[] entryNames) { ArgumentNullException.ThrowIfNull(archive); @@ -138,182 +141,62 @@ public void UpdateAllocationAfterMove(Stream image, string fileName, long oldOff // ── IArchiveDefragmentable ─────────────────────────────────────────── /// - /// Packs live payloads immediately after the directory with native byte moves. + /// Performs the defragment operation. /// public void Defragment(Stream archive) => Defragment(archive, new DefragOptions { Mode = DefragMode.ConsolidateAtStart }); /// - /// Defragments a T64 image. The canonical consolidate-at-start path moves - /// normal payloads in place and patches only their absolute data-offset fields. - /// Other layout modes retain the established rebuild behaviour, but preserve - /// T64 version, tape name, load address and Commodore file type. + /// Defragments a T64 image. Falls back to rebuild since T64 data offsets + /// are stored in directory entries and recompaction is simplest via rebuild. /// - public void Defragment(Stream archive, DefragOptions options) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(options); - EnsureMutable(archive); - options.CancellationToken.ThrowIfCancellationRequested(); - - if (options.Mode != DefragMode.ConsolidateAtStart) { - DefragmentByRebuild(archive, options); - return; - } - - archive.Position = 0; - using var reader = new T64Reader(archive); - if (reader.Entries.Any(static e => e.EntryType != 1)) - throw new InvalidDataException("T64: moving defrag only supports normal directory records (entry type 1)."); - - var entries = reader.Entries.OrderBy(static e => e.DataOffset).ToArray(); - var imageSize = archive.Length; - var cursor = (long)HeaderSize + reader.DirectoryEntryCount * EntrySize; - var mover = new T64BlockMover(); - - options.OnProgress?.Invoke(new DefragProgressEvent( - Phase: "scanning", Fraction: 0, CurrentReadOffset: 0, CurrentWriteOffset: -1, - ImageSize: imageSize, BlockMap: EnumerateLayout(archive).ToList(), Status: "Analysing T64 layout")); - - for (var i = 0; i < entries.Length; i++) { - options.CancellationToken.ThrowIfCancellationRequested(); - var entry = entries[i]; - if (entry.DataOffset != cursor) { - if (entry.Size > 0) - mover.MoveExtent(archive, entry.DataOffset, cursor, entry.Size, zeroSource: true); - mover.UpdateAllocationAfterMove(archive, entry.Name, entry.DataOffset, cursor, entry.Size); - } - cursor += entry.Size; - - options.OnProgress?.Invoke(new DefragProgressEvent( - Phase: "moving", Fraction: entries.Length == 0 ? 1 : (double)(i + 1) / entries.Length, - CurrentReadOffset: entry.DataOffset, CurrentWriteOffset: cursor, - ImageSize: imageSize, BlockMap: null, Status: $"Packed {entry.Name}")); - } - - archive.SetLength(cursor); - archive.Position = 0; - options.OnProgress?.Invoke(new DefragProgressEvent( - Phase: "complete", Fraction: 1, CurrentReadOffset: -1, CurrentWriteOffset: -1, - ImageSize: archive.Length, BlockMap: EnumerateLayout(archive).ToList(), Status: "Defragmentation complete")); - } - - // ── IArchivePurgeable ──────────────────────────────────────────────── - - /// - /// Wipes all directory/payload bytes and leaves the canonical 64-byte empty T64 header. - /// - public void Purge(Stream archive) { - ArgumentNullException.ThrowIfNull(archive); - EnsureMutable(archive); + public void Defragment(Stream archive, DefragOptions options) + => DefragRebuilder.Rebuild(archive, options, ReadEntries, BuildImage); - archive.Position = 0; - using (var reader = new T64Reader(archive)) { } - - if (archive.Length > HeaderSize) - ZeroRange(archive, HeaderSize, archive.Length - HeaderSize); - - Span counts = stackalloc byte[4]; - archive.Position = 34; - archive.Write(counts); - archive.SetLength(HeaderSize); - archive.Position = 0; - } - - // ── IArchiveLayoutMap / IWipeEmpty ────────────────────────────────── + // ── IArchiveLayoutMap ──────────────────────────────────────────────── /// - /// Enumerates every byte of a normal-record T64 image: the fixed header, live - /// and free directory slots, live payloads, and proven dead gaps/tail bytes. - /// Unknown record kinds fail closed by exposing no free-space map at all. + /// Enumerates the byte layout of a T64 tape image: 64-byte header as + /// MetadataReserved, N×32-byte directory entries as MetadataReserved, + /// and each file's data region as Used. /// public IEnumerable EnumerateLayout(Stream archive) { - ArgumentNullException.ThrowIfNull(archive); archive.Position = 0; - using var reader = new T64Reader(archive); - if (reader.Entries.Any(static e => e.EntryType != 1)) - yield break; - - var liveBySlot = reader.Entries.ToDictionary(static e => e.DirectoryIndex); - - yield return new DefragBlockInfo(0, HeaderSize, DefragBlockKind.MetadataReserved, "T64 Header"); - - for (var i = 0; i < reader.DirectoryEntryCount; i++) { - var offset = HeaderSize + (long)i * EntrySize; - if (liveBySlot.TryGetValue(i, out var entry)) - yield return new DefragBlockInfo(offset, EntrySize, DefragBlockKind.MetadataReserved, $"Directory: {entry.Name}"); - else - yield return new DefragBlockInfo(offset, EntrySize, DefragBlockKind.Free); - } + var r = new T64Reader(archive); - var cursor = (long)HeaderSize + reader.DirectoryEntryCount * EntrySize; - foreach (var entry in reader.Entries.OrderBy(static e => e.DataOffset)) { - if (entry.DataOffset > cursor) - yield return new DefragBlockInfo(cursor, entry.DataOffset - cursor, DefragBlockKind.Free); + // Header: 64 bytes + const int headerSize = 64; + const int entrySize = 32; + var dirSize = r.Entries.Count * entrySize; - if (entry.Size > 0) - yield return new DefragBlockInfo(entry.DataOffset, entry.Size, DefragBlockKind.Used, entry.Name); + yield return new DefragBlockInfo(0, headerSize, DefragBlockKind.MetadataReserved, "T64 Header"); + if (dirSize > 0) + yield return new DefragBlockInfo(headerSize, dirSize, DefragBlockKind.MetadataReserved, "Directory"); - cursor = Math.Max(cursor, entry.DataOffset + entry.Size); + foreach (var e in r.Entries) { + if (e.Size > 0) + yield return new DefragBlockInfo(e.DataOffset, e.Size, DefragBlockKind.Used, e.Name); } - if (cursor < archive.Length) - yield return new DefragBlockInfo(cursor, archive.Length - cursor, DefragBlockKind.Free); + var imageSize = archive.Length; + var dataEnd = r.Entries.Count > 0 + ? r.Entries.Max(e => e.DataOffset + e.Size) + : headerSize + dirSize; + if (dataEnd < imageSize) + yield return new DefragBlockInfo(dataEnd, imageSize - dataEnd, DefragBlockKind.Free); } - private static void DefragmentByRebuild(Stream archive, DefragOptions options) { - archive.Position = 0; - using var reader = new T64Reader(archive); - if (reader.Entries.Any(static e => e.EntryType != 1)) - throw new InvalidDataException("T64: rebuild defrag only supports normal directory records (entry type 1)."); - - var tapeName = reader.TapeName; - var version = reader.Version; - var metadata = reader.Entries.Select(e => new SnapshotEntry( - e.Name, e.StartAddress, e.FileType, reader.Extract(e))).ToList(); - - DefragRebuilder.Rebuild(archive, options, ReadEntries, files => { - var remaining = metadata.ToList(); - var writer = new T64Writer(); - foreach (var (name, data) in files) { - var index = remaining.FindIndex(e => - e.Name.Equals(name, StringComparison.Ordinal) && e.Data.AsSpan().SequenceEqual(data)); - if (index < 0) { - writer.AddFile(name, data); - continue; - } - - var source = remaining[index]; - remaining.RemoveAt(index); - writer.AddFile(name, source.StartAddress, source.FileType, data); - } - return writer.Build(tapeName, version); - }); - } + // ── Shared delegates ───────────────────────────────────────────────── private static IEnumerable<(string Name, byte[] Data)> ReadEntries(Stream stream) { - using var reader = new T64Reader(stream); - return reader.Entries.Select(e => (e.Name, reader.Extract(e))).ToArray(); + var r = new T64Reader(stream); + return r.Entries.Select(e => (e.Name, r.Extract(e))); } - private static void EnsureMutable(Stream archive) { - if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) - throw new ArgumentException("T64: stream must be readable, writable and seekable.", nameof(archive)); + private static byte[] BuildImage(IReadOnlyList<(string Name, byte[] Data)> files) { + var w = new T64Writer(); + foreach (var (n, d) in files) + w.AddFile(n.Length > 16 ? n[..16] : n, d); + return w.Build(); } - - private static void ZeroRange(Stream stream, long offset, long length) { - var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); - try { - Array.Clear(buffer, 0, buffer.Length); - stream.Position = offset; - while (length > 0) { - var chunk = (int)Math.Min(length, buffer.Length); - stream.Write(buffer, 0, chunk); - length -= chunk; - } - } finally { - ArrayPool.Shared.Return(buffer); - } - } - - private sealed record SnapshotEntry(string Name, ushort StartAddress, byte FileType, byte[] Data); } diff --git a/FileFormats/FileFormat.T64/T64InPlaceModifier.cs b/FileFormats/FileFormat.T64/T64InPlaceModifier.cs index 266978a74..5af77eaa2 100644 --- a/FileFormats/FileFormat.T64/T64InPlaceModifier.cs +++ b/FileFormats/FileFormat.T64/T64InPlaceModifier.cs @@ -15,23 +15,29 @@ namespace FileFormat.T64; /// /// 0..63 - 64-byte header (signature, version, max-entries, used-entries, tape name). /// 64..64+maxEntries*32 - directory: N * 32-byte slot records (entry type, C64 type, start/end addr, absolute data offset, filename). -/// 64+maxEntries*32 .. EOF - file payloads addressed by each slot's absolute dataOffset field. +/// 64+maxEntries*32 .. EOF - concatenated file payloads addressed by each slot's absolute dataOffset field. /// /// Add: if a directory slot is currently free (entryType=0) the /// new entry fills that slot and the payload is appended at EOF. If the /// directory is full the directory grows by one 32-byte slot: every file /// payload shifts forward by 32 bytes, every existing slot's absolute /// dataOffset field is patched by +32, then the new slot is written and -/// the new payload appended at the new EOF. -/// Remove: shifts the later directory slots up by 32 bytes, wipes -/// the removed payload, closes the payload gap, patches all surviving absolute -/// data offsets, and truncates the stream. +/// the new payload appended at the new EOF. Header's maxEntries and +/// usedEntries are updated to match. +/// Remove: shifts the later directory slots up by 32 bytes inside +/// the directory table (the vacated trailing slot is zero-filled to leave no +/// forensic trace), then wipes the removed file's payload bytes and shifts the +/// remaining payload region into the vacated payload range (updating each +/// affected slot's dataOffset field). The stream is truncated to the +/// new EOF. Both maxEntries and usedEntries are decremented so +/// the directory remains exactly sized. /// public static class T64InPlaceModifier { private const int HeaderSize = 64; private const int EntrySize = 32; private const int MaxEntriesOffset = 34; + private const int UsedEntriesOffset = 36; /// /// Adds (or replaces by name, case-insensitive) a single file inside an @@ -41,57 +47,55 @@ public static void AddFile(Stream image, string name, byte[] data, ushort startA ArgumentNullException.ThrowIfNull(image); ArgumentNullException.ThrowIfNull(name); ArgumentNullException.ThrowIfNull(data); - EnsureMutable(image); - ValidatePayload(name, startAddress, data.Length); - - // Validate the image before replacement semantics can remove anything. - image.Position = 0; - using (var reader = new T64Reader(image)) { } + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("T64: stream must be readable, writable and seekable.", nameof(image)); + // Replace-by-name semantic: drop any prior entry with the same name first. var existingIndex = FindEntryIndex(image, name); if (existingIndex >= 0) RemoveEntryAt(image, existingIndex); - var (maxEntries, _) = ReadHeaderCounts(image); - var usedEntries = CountUsedEntries(image, maxEntries); + // Re-read header after possible removal. + var (maxEntries, usedEntries) = ReadHeaderCounts(image); + // Try to find a free slot first. var freeSlot = FindFreeSlot(image, maxEntries); if (freeSlot >= 0) { AppendDataAndFillSlot(image, freeSlot, name, data, startAddress); - WriteHeaderCounts(image, maxEntries, checked((ushort)(usedEntries + 1))); + WriteHeaderCounts(image, maxEntries, (ushort)(usedEntries + 1)); return; } - if (maxEntries == ushort.MaxValue) - throw new IOException("T64: directory cannot grow beyond 65535 slots."); - - // Directory is full: insert one slot immediately before the payload region. + // Directory is full → grow by one slot. Shift payload region right by 32 + // bytes and patch every existing slot's absolute dataOffset by +32. var payloadStart = (long)HeaderSize + maxEntries * EntrySize; var payloadLength = image.Length - payloadStart; ShiftRangeForward(image, payloadStart, payloadLength, EntrySize); + // Zero-init the new slot's bytes so any future scan sees a clean slot. Span zero = stackalloc byte[EntrySize]; image.Position = payloadStart; image.Write(zero); + // Patch every existing slot's absolute dataOffset += EntrySize. Span entry = stackalloc byte[EntrySize]; for (var i = 0; i < maxEntries; i++) { var slotOff = HeaderSize + i * EntrySize; image.Position = slotOff; image.ReadExactly(entry); if (entry[0] == 0) continue; - var oldDataOff = BinaryPrimitives.ReadUInt32LittleEndian(entry[8..]); - if (oldDataOff > uint.MaxValue - EntrySize) - throw new InvalidDataException("T64: payload offset cannot be shifted into the 32-bit offset field."); - BinaryPrimitives.WriteUInt32LittleEndian(entry[8..], oldDataOff + EntrySize); + var oldDataOff = BinaryPrimitives.ReadUInt32LittleEndian(entry.Slice(8)); + BinaryPrimitives.WriteUInt32LittleEndian(entry.Slice(8), oldDataOff + (uint)EntrySize); image.Position = slotOff; image.Write(entry); } + // Write new slot at the freed gap (the now-zeroed bytes at payloadStart). var newSlotIndex = maxEntries; - var newMaxEntries = checked((ushort)(maxEntries + 1)); + var newMaxEntries = (ushort)(maxEntries + 1); AppendDataAndFillSlot(image, newSlotIndex, name, data, startAddress); - WriteHeaderCounts(image, newMaxEntries, checked((ushort)(usedEntries + 1))); + + WriteHeaderCounts(image, newMaxEntries, (ushort)(usedEntries + 1)); } /// @@ -101,10 +105,8 @@ public static void AddFile(Stream image, string name, byte[] data, ushort startA public static bool RemoveFile(Stream image, string name) { ArgumentNullException.ThrowIfNull(image); ArgumentNullException.ThrowIfNull(name); - EnsureMutable(image); - - image.Position = 0; - using (var reader = new T64Reader(image)) { } + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("T64: stream must be readable, writable and seekable.", nameof(image)); var index = FindEntryIndex(image, name); if (index < 0) return false; @@ -112,20 +114,26 @@ public static bool RemoveFile(Stream image, string name) { return true; } + // ── Internals ──────────────────────────────────────────────────────────── + private static void RemoveEntryAt(Stream image, int index) { - var (maxEntries, _) = ReadHeaderCounts(image); + var (maxEntries, usedEntries) = ReadHeaderCounts(image); if (index < 0 || index >= maxEntries) return; - image.Position = 0; - using var reader = new T64Reader(image); - var removed = reader.Entries.FirstOrDefault(e => e.DirectoryIndex == index); - if (removed is null) return; - - var removedDataOffset = (long)removed.DataOffset; - var removedDataLen = removed.Size; - var usedEntries = reader.Entries.Count; - - if (removedDataLen > 0) + // Read the slot we're removing to learn the data extent. + Span entry = stackalloc byte[EntrySize]; + var slotOff = HeaderSize + index * EntrySize; + image.Position = slotOff; + image.ReadExactly(entry); + + var startAddr = BinaryPrimitives.ReadUInt16LittleEndian(entry.Slice(2)); + var endAddr = BinaryPrimitives.ReadUInt16LittleEndian(entry.Slice(4)); + var removedDataOffset = (long)BinaryPrimitives.ReadUInt32LittleEndian(entry.Slice(8)); + var removedDataLen = endAddr > startAddr ? endAddr - startAddr : 0; + var wasOccupied = entry[0] != 0; + + // Wipe the removed payload bytes for the forensic guarantee. + if (wasOccupied && removedDataLen > 0) ZeroRange(image, removedDataOffset, removedDataLen); // 1) Compact directory: shift later slots up by 32 bytes. @@ -134,66 +142,79 @@ private static void RemoveEntryAt(Stream image, int index) { if (laterSlotsLen > 0) ShiftRangeBackward(image, afterEntryOffset, laterSlotsLen, EntrySize); - // 2) Compact payload bytes. Every payload loses the 32-byte directory slot; - // payloads after the removed member additionally lose that member's bytes. + // 2) Compact payload region: the directory shrank by 32 bytes, so all file + // payloads shift back by 32. Additionally, the removed payload's bytes + // must be excised. + // Old payload region: [oldPayloadStart .. image.Length) + // Within it, removedDataOffset .. removedDataOffset+removedDataLen is gone. + // New payload region: shifted back by 32 bytes, and the removed slice is closed up. var oldPayloadStart = (long)HeaderSize + maxEntries * EntrySize; + var newPayloadStart = oldPayloadStart - EntrySize; var oldEnd = image.Length; - var beforeRemovedLength = removedDataOffset - oldPayloadStart; - if (beforeRemovedLength > 0) - ShiftRangeBackward(image, oldPayloadStart, beforeRemovedLength, EntrySize); - - var afterRemovedSource = checked(removedDataOffset + removedDataLen); - var afterRemovedLength = oldEnd - afterRemovedSource; - if (afterRemovedLength > 0) - ShiftRangeBackward(image, afterRemovedSource, afterRemovedLength, EntrySize + removedDataLen); - - // 3) Patch surviving absolute payload offsets after the directory move. - Span entry = stackalloc byte[EntrySize]; - var newMaxEntries = checked((ushort)(maxEntries - 1)); + // Sub-region A: [oldPayloadStart .. removedDataOffset) shifts back by 32. + var aLen = (wasOccupied ? removedDataOffset : oldEnd) - oldPayloadStart; + if (aLen > 0) + ShiftRangeBackward(image, oldPayloadStart, aLen, EntrySize); + + // Sub-region B: [removedDataOffset+removedDataLen .. oldEnd) shifts back by + // (32 + removedDataLen) so the removed-payload slice closes up. + var bSrc = removedDataOffset + removedDataLen; + var bLen = oldEnd - bSrc; + if (wasOccupied && bLen > 0) + ShiftRangeBackward(image, bSrc, bLen, EntrySize + removedDataLen); + + // 3) Patch every remaining slot's absolute dataOffset. + // Slots whose dataOffset was < removedDataOffset shift by -32. + // Slots whose dataOffset was > removedDataOffset shift by -(32 + removedDataLen). + // (The removed slot itself is already gone after directory compaction.) + var newMaxEntries = (ushort)(maxEntries - 1); for (var i = 0; i < newMaxEntries; i++) { - var slotOff = HeaderSize + i * EntrySize; - image.Position = slotOff; + var so = HeaderSize + i * EntrySize; + image.Position = so; image.ReadExactly(entry); if (entry[0] == 0) continue; - - var dataOff = (long)BinaryPrimitives.ReadUInt32LittleEndian(entry[8..]); - var newDataOff = dataOff < removedDataOffset - ? dataOff - EntrySize - : dataOff - EntrySize - removedDataLen; - if (newDataOff < HeaderSize + (long)newMaxEntries * EntrySize || newDataOff > uint.MaxValue) - throw new InvalidDataException("T64: compaction produced an invalid payload offset."); - - BinaryPrimitives.WriteUInt32LittleEndian(entry[8..], (uint)newDataOff); - image.Position = slotOff; + var dataOff = (long)BinaryPrimitives.ReadUInt32LittleEndian(entry.Slice(8)); + long newDataOff; + if (!wasOccupied || dataOff < removedDataOffset) + newDataOff = dataOff - EntrySize; + else + newDataOff = dataOff - EntrySize - removedDataLen; + BinaryPrimitives.WriteUInt32LittleEndian(entry.Slice(8), (uint)newDataOff); + image.Position = so; image.Write(entry); } - // 4) Truncate and repair both header counts from the actual live directory. - var newEnd = oldEnd - EntrySize - removedDataLen; + // 4) Truncate the stream to the new EOF. + var newEnd = oldEnd - EntrySize - (wasOccupied ? removedDataLen : 0); image.SetLength(newEnd); - WriteHeaderCounts(image, newMaxEntries, checked((ushort)(usedEntries - 1))); + + // 5) Patch header counts. + var newUsedEntries = wasOccupied && usedEntries > 0 ? (ushort)(usedEntries - 1) : usedEntries; + WriteHeaderCounts(image, newMaxEntries, newUsedEntries); } + /// + /// Writes data at EOF, then fills the given directory slot with type, + /// addresses, dataOffset and filename. + /// private static void AppendDataAndFillSlot(Stream image, int slotIndex, string name, byte[] data, ushort startAddress) { var dataOffset = image.Length; - if (dataOffset > uint.MaxValue) - throw new IOException("T64: payload offset exceeds the 32-bit directory field."); - image.Position = dataOffset; image.Write(data); Span entry = stackalloc byte[EntrySize]; - entry[0] = 1; - entry[1] = 0x82; - BinaryPrimitives.WriteUInt16LittleEndian(entry[2..], startAddress); - BinaryPrimitives.WriteUInt16LittleEndian(entry[4..], unchecked((ushort)(startAddress + data.Length))); - BinaryPrimitives.WriteUInt32LittleEndian(entry[8..], (uint)dataOffset); + entry[0] = 1; // entry type: normal + entry[1] = 0x82; // C64 file type: PRG + BinaryPrimitives.WriteUInt16LittleEndian(entry.Slice(2), startAddress); + BinaryPrimitives.WriteUInt16LittleEndian(entry.Slice(4), (ushort)(startAddress + data.Length)); + BinaryPrimitives.WriteUInt32LittleEndian(entry.Slice(8), (uint)dataOffset); var trimmed = name.Length > 16 ? name[..16] : name; var nameBytes = Encoding.ASCII.GetBytes(trimmed); - nameBytes.CopyTo(entry[16..]); - entry[(16 + nameBytes.Length)..32].Fill(0x20); + nameBytes.CopyTo(entry.Slice(16)); + for (var j = nameBytes.Length; j < 16; j++) + entry[16 + j] = 0x20; image.Position = HeaderSize + slotIndex * EntrySize; image.Write(entry); @@ -202,7 +223,8 @@ private static void AppendDataAndFillSlot(Stream image, int slotIndex, string na private static int FindFreeSlot(Stream image, int maxEntries) { for (var i = 0; i < maxEntries; i++) { image.Position = HeaderSize + i * EntrySize; - if (image.ReadByte() == 0) return i; + var typeByte = image.ReadByte(); + if (typeByte == 0) return i; } return -1; } @@ -215,114 +237,101 @@ private static int FindEntryIndex(Stream image, string name) { image.Position = HeaderSize + i * EntrySize; image.ReadExactly(entry); if (entry[0] == 0) continue; - var entryName = Encoding.ASCII.GetString(entry[16..32]).TrimEnd('\0', ' '); + var entryName = Encoding.ASCII.GetString(entry.Slice(16, 16)).TrimEnd('\0', ' '); if (entryName.Equals(trimmed, StringComparison.OrdinalIgnoreCase)) return i; } return -1; } - private static int CountUsedEntries(Stream image, int maxEntries) { - var result = 0; - for (var i = 0; i < maxEntries; i++) { - image.Position = HeaderSize + i * EntrySize; - if (image.ReadByte() > 0) ++result; - } - return result; - } - private static (ushort MaxEntries, ushort UsedEntries) ReadHeaderCounts(Stream image) { if (image.Length < HeaderSize) throw new InvalidDataException("T64: stream too small."); - Span buffer = stackalloc byte[4]; + Span buf = stackalloc byte[4]; image.Position = MaxEntriesOffset; - image.ReadExactly(buffer); - var maxEntries = BinaryPrimitives.ReadUInt16LittleEndian(buffer); - var usedEntries = BinaryPrimitives.ReadUInt16LittleEndian(buffer[2..]); - if ((long)HeaderSize + maxEntries * EntrySize > image.Length) - throw new InvalidDataException("T64: directory extends beyond end of image."); - return (maxEntries, usedEntries); + image.ReadExactly(buf); + return ( + BinaryPrimitives.ReadUInt16LittleEndian(buf), + BinaryPrimitives.ReadUInt16LittleEndian(buf.Slice(2))); } private static void WriteHeaderCounts(Stream image, ushort maxEntries, ushort usedEntries) { - Span buffer = stackalloc byte[4]; - BinaryPrimitives.WriteUInt16LittleEndian(buffer, maxEntries); - BinaryPrimitives.WriteUInt16LittleEndian(buffer[2..], usedEntries); + Span buf = stackalloc byte[4]; + BinaryPrimitives.WriteUInt16LittleEndian(buf, maxEntries); + BinaryPrimitives.WriteUInt16LittleEndian(buf.Slice(2), usedEntries); image.Position = MaxEntriesOffset; - image.Write(buffer); - } - - private static void EnsureMutable(Stream image) { - if (!image.CanRead || !image.CanWrite || !image.CanSeek) - throw new ArgumentException("T64: stream must be readable, writable and seekable.", nameof(image)); - } - - private static void ValidatePayload(string name, ushort startAddress, int length) { - if (startAddress + (long)length > 0x10000) - throw new InvalidOperationException( - $"T64: '{name}' is {length:N0} bytes and loads at ${startAddress:X4}, past the C64 64 KB address space."); + image.Write(buf); } private static void ZeroRange(Stream image, long offset, long length) { if (length <= 0) return; - var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); + var buf = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); try { - Array.Clear(buffer, 0, buffer.Length); + Array.Clear(buf); var remaining = length; image.Position = offset; while (remaining > 0) { - var chunk = (int)Math.Min(remaining, buffer.Length); - image.Write(buffer, 0, chunk); + var chunk = (int)Math.Min(remaining, buf.Length); + image.Write(buf, 0, chunk); remaining -= chunk; } } finally { - ArrayPool.Shared.Return(buffer); + ArrayPool.Shared.Return(buf); } } + /// + /// Shifts bytes [src .. src+length) to [src+delta .. src+delta+length), where + /// delta > 0. Copies high-to-low so overlap doesn't corrupt the data. + /// private static void ShiftRangeForward(Stream image, long src, long length, long delta) { if (length <= 0 || delta == 0) return; if (delta < 0) throw new ArgumentOutOfRangeException(nameof(delta)); - var dstEnd = checked(src + delta + length); + var dstEnd = src + delta + length; if (dstEnd > image.Length) image.SetLength(dstEnd); - var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); + var buf = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); try { var remaining = length; while (remaining > 0) { - var chunk = (int)Math.Min(remaining, buffer.Length); + var chunk = (int)Math.Min(remaining, buf.Length); var readFrom = src + remaining - chunk; + var writeTo = readFrom + delta; image.Position = readFrom; - image.ReadExactly(buffer, 0, chunk); - image.Position = readFrom + delta; - image.Write(buffer, 0, chunk); + image.ReadExactly(buf, 0, chunk); + image.Position = writeTo; + image.Write(buf, 0, chunk); remaining -= chunk; } } finally { - ArrayPool.Shared.Return(buffer); + ArrayPool.Shared.Return(buf); } } + /// + /// Shifts bytes [src .. src+length) to [src-delta .. src-delta+length), where + /// delta > 0. Copies low-to-high since destination < source. + /// private static void ShiftRangeBackward(Stream image, long src, long length, long delta) { if (length <= 0 || delta == 0) return; if (delta < 0) throw new ArgumentOutOfRangeException(nameof(delta)); - var buffer = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); + var buf = ArrayPool.Shared.Rent((int)Math.Min(length, 64 * 1024)); try { var remaining = length; var cursor = 0L; while (remaining > 0) { - var chunk = (int)Math.Min(remaining, buffer.Length); + var chunk = (int)Math.Min(remaining, buf.Length); image.Position = src + cursor; - image.ReadExactly(buffer, 0, chunk); + image.ReadExactly(buf, 0, chunk); image.Position = src + cursor - delta; - image.Write(buffer, 0, chunk); + image.Write(buf, 0, chunk); cursor += chunk; remaining -= chunk; } } finally { - ArrayPool.Shared.Return(buffer); + ArrayPool.Shared.Return(buf); } } } diff --git a/FileFormats/FileFormat.T64/T64Modifier.cs b/FileFormats/FileFormat.T64/T64Modifier.cs index fb3216418..b99041e76 100644 --- a/FileFormats/FileFormat.T64/T64Modifier.cs +++ b/FileFormats/FileFormat.T64/T64Modifier.cs @@ -1,20 +1,129 @@ #pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; namespace FileFormat.T64; /// -/// Compatibility facade for the T64 in-place editor. +/// In-place T64 modifier — performs O(touched bytes) random-access I/O against a +/// T64 tape image. T64 has a 64-byte header followed by a fixed-size directory +/// table of N×32-byte slots, then concatenated file data. +/// +/// AddFile: finds an empty slot (entryType=0) in the directory, +/// appends file data at EOF, and fills in the slot. +/// RemoveFile: sets the slot's entryType to 0 (marks it free). +/// Data is left in place (no compaction). /// public static class T64Modifier { + + private const int HeaderSize = 64; + private const int EntrySize = 32; + /// - /// Adds or replaces a file in an existing T64 tape image. + /// Adds a file to an existing T64 tape image. Finds the first free slot + /// (entryType=0) in the directory, appends the file data at the end of the + /// image, and writes the directory entry. /// - public static void AddFile(Stream image, string name, byte[] data, ushort startAddress = 0x0801) - => T64InPlaceModifier.AddFile(image, name, data, startAddress); + public static void AddFile(Stream image, string name, byte[] data, ushort startAddress = 0x0801) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(data); + + // Read header to get maxEntries. + image.Position = 34; + Span hdrBuf = stackalloc byte[4]; + image.ReadExactly(hdrBuf); + var maxEntries = BinaryPrimitives.ReadUInt16LittleEndian(hdrBuf); + var usedEntries = BinaryPrimitives.ReadUInt16LittleEndian(hdrBuf.Slice(2)); + + // Scan the directory for a free slot. + var freeSlot = -1; + for (var i = 0; i < maxEntries; i++) { + var slotOff = HeaderSize + i * EntrySize; + image.Position = slotOff; + var typeByte = image.ReadByte(); + if (typeByte <= 0) { // 0 or EOF + freeSlot = i; + break; + } + } + + if (freeSlot < 0) + throw new IOException($"T64: no free directory slot (max {maxEntries})."); + + // Append data at EOF. + var dataOffset = (int)image.Length; + image.Position = dataOffset; + image.Write(data); + + // Build directory entry (32 bytes). + var entry = new byte[EntrySize]; + entry[0] = 1; // entryType = normal + entry[1] = 0x82; // C64 file type = PRG + + BinaryPrimitives.WriteUInt16LittleEndian(entry.AsSpan(2), startAddress); + BinaryPrimitives.WriteUInt16LittleEndian(entry.AsSpan(4), (ushort)(startAddress + data.Length)); + BinaryPrimitives.WriteUInt32LittleEndian(entry.AsSpan(8), (uint)dataOffset); + + // Filename: 16 bytes, space-padded. + var nameBytes = Encoding.ASCII.GetBytes(name.Length > 16 ? name[..16] : name); + nameBytes.CopyTo(entry, 16); + for (var j = nameBytes.Length; j < 16; j++) + entry[16 + j] = 0x20; + + // Write the directory entry. + image.Position = HeaderSize + freeSlot * EntrySize; + image.Write(entry); + + // Update usedEntries count. + usedEntries++; + image.Position = 36; + Span usedBuf = stackalloc byte[2]; + BinaryPrimitives.WriteUInt16LittleEndian(usedBuf, usedEntries); + image.Write(usedBuf); + } /// - /// Removes a named file from the T64 image and compacts the vacated bytes. + /// Removes a named file from the T64 image by zeroing its directory entry type. + /// Returns false if not found. /// - public static bool RemoveFile(Stream image, string name) - => T64InPlaceModifier.RemoveFile(image, name); + public static bool RemoveFile(Stream image, string name) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(name); + + image.Position = 34; + Span hdrBuf = stackalloc byte[4]; + image.ReadExactly(hdrBuf); + var maxEntries = BinaryPrimitives.ReadUInt16LittleEndian(hdrBuf); + var usedEntries = BinaryPrimitives.ReadUInt16LittleEndian(hdrBuf.Slice(2)); + + for (var i = 0; i < maxEntries; i++) { + var slotOff = HeaderSize + i * EntrySize; + var entry = new byte[EntrySize]; + image.Position = slotOff; + image.ReadExactly(entry); + + if (entry[0] == 0) continue; // free slot + + // Read filename (16 bytes at offset 16). + var entryName = Encoding.ASCII.GetString(entry, 16, 16).TrimEnd('\0', ' '); + if (!entryName.Equals(name, StringComparison.OrdinalIgnoreCase)) continue; + + // Mark slot as free. + entry[0] = 0; + image.Position = slotOff; + image.Write(entry); + + // Decrement used count. + if (usedEntries > 0) usedEntries--; + image.Position = 36; + Span usedBuf = stackalloc byte[2]; + BinaryPrimitives.WriteUInt16LittleEndian(usedBuf, usedEntries); + image.Write(usedBuf); + + return true; + } + + return false; + } } diff --git a/FileFormats/FileFormat.T64/T64Reader.cs b/FileFormats/FileFormat.T64/T64Reader.cs index 335697993..51372fe51 100644 --- a/FileFormats/FileFormat.T64/T64Reader.cs +++ b/FileFormats/FileFormat.T64/T64Reader.cs @@ -16,39 +16,17 @@ public sealed class T64Reader : IDisposable { /// public IReadOnlyList Entries => _entries; /// - /// Gets the T64 format version. - /// - public ushort Version { get; private set; } - /// - /// Gets the number of directory slots reserved by the image. - /// - public ushort DirectoryEntryCount { get; private set; } - /// - /// Gets the used-entry count stored in the header. - /// - public ushort UsedEntryCount { get; private set; } - /// /// Gets or sets the tape name. /// public string TapeName { get; private set; } = ""; private const int HeaderSize = 64; private const int EntrySize = 32; - private const ushort Conv64BrokenEndAddress = 0xC3C6; /// /// Initializes a new instance of . /// public T64Reader(Stream stream, bool leaveOpen = false) { - ArgumentNullException.ThrowIfNull(stream); - if (!stream.CanRead) - throw new ArgumentException("T64: stream must be readable.", nameof(stream)); - if (stream.CanSeek) { - if (stream.Length > int.MaxValue) - throw new InvalidDataException("T64: images larger than 2 GiB are not supported by the in-memory reader."); - stream.Position = 0; - } - using var ms = new MemoryStream(); stream.CopyTo(ms); _data = ms.ToArray(); @@ -59,122 +37,67 @@ private void Parse() { if (_data.Length < HeaderSize) throw new InvalidDataException("T64: file too small."); - var signature = Encoding.ASCII.GetString(_data, 0, 32).TrimEnd('\0', ' '); - if (!IsKnownSignature(signature)) + // Validate magic + var sig = Encoding.ASCII.GetString(_data, 0, 3); + if (sig != "C64") throw new InvalidDataException("T64: invalid magic."); - Version = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(32)); - DirectoryEntryCount = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(34)); - UsedEntryCount = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(36)); - - var directoryEnd = checked(HeaderSize + DirectoryEntryCount * EntrySize); - if (directoryEnd > _data.Length) - throw new InvalidDataException("T64: directory extends beyond end of image."); + var maxEntries = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(34)); + var usedEntries = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(36)); // Tape name (24 bytes at offset 40) TapeName = Encoding.ASCII.GetString(_data, 40, 24).TrimEnd('\0', ' '); - var rawEntries = new List(); - for (var i = 0; i < DirectoryEntryCount; i++) { + // Read directory entries + for (var i = 0; i < maxEntries; i++) { var off = HeaderSize + i * EntrySize; + if (off + EntrySize > _data.Length) break; + var entryType = _data[off]; - if (entryType == 0) continue; + if (entryType == 0) continue; // free slot - var fileType = _data[off + 1]; var startAddr = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(off + 2)); var endAddr = BinaryPrimitives.ReadUInt16LittleEndian(_data.AsSpan(off + 4)); - var rawDataOffset = BinaryPrimitives.ReadUInt32LittleEndian(_data.AsSpan(off + 8)); - if (rawDataOffset > int.MaxValue) - throw new InvalidDataException($"T64: entry {i} data offset is outside the supported image range."); - var dataOffset = (int)rawDataOffset; - if (dataOffset < directoryEnd || dataOffset > _data.Length) - throw new InvalidDataException($"T64: entry {i} data offset is outside the payload region."); + var dataOffset = (int)BinaryPrimitives.ReadUInt32LittleEndian(_data.AsSpan(off + 8)); + // Filename: 16 bytes at offset 16 within entry var nameBytes = _data.AsSpan(off + 16, 16); - var nameLength = 16; - while (nameLength > 0 && nameBytes[nameLength - 1] is 0x00 or 0x20) - --nameLength; - var name = Encoding.ASCII.GetString(nameBytes[..nameLength]); - - rawEntries.Add(new RawEntry(i, entryType, fileType, startAddr, endAddr, dataOffset, name)); - } + var nameEnd = 16; + for (var j = 15; j >= 0; j--) { + if (nameBytes[j] != 0x20 && nameBytes[j] != 0x00) { nameEnd = j + 1; break; } + if (j == 0) nameEnd = 0; + } + var name = Encoding.ASCII.GetString(_data, off + 16, nameEnd); - var ordered = rawEntries.OrderBy(static e => e.DataOffset).ToArray(); - for (var i = 0; i < ordered.Length; i++) { - var raw = ordered[i]; - var nextOffset = i + 1 < ordered.Length ? ordered[i + 1].DataOffset : _data.Length; - if (nextOffset < raw.DataOffset) - throw new InvalidDataException($"T64: entry {raw.DirectoryIndex} has an invalid payload order."); - - var physicalBytesAvailable = nextOffset - raw.DataOffset; - var size = ResolveSize(raw, physicalBytesAvailable); - if (size < 0 || size > physicalBytesAvailable) - throw new InvalidDataException($"T64: entry {raw.DirectoryIndex} extends into the next payload or past EOF."); + var size = endAddr > startAddr ? endAddr - startAddr : 0; _entries.Add(new T64Entry { - DirectoryIndex = raw.DirectoryIndex, - Name = raw.Name, + Name = name, Size = size, - EntryType = raw.EntryType, - FileType = raw.FileType, - StartAddress = raw.StartAddress, - EndAddress = raw.EndAddress, - DataOffset = raw.DataOffset, + EntryType = entryType, + StartAddress = startAddr, + EndAddress = endAddr, + DataOffset = dataOffset, }); } - - _entries.Sort(static (a, b) => a.DirectoryIndex.CompareTo(b.DirectoryIndex)); } - private static long ResolveSize(RawEntry entry, int physicalBytesAvailable) { - // CONV64 historically wrote $C3C6 into every end-address field. VICE and - // t64fix recover such records from the next payload offset (or EOF for the - // final entry), so mirror that public behaviour without depending on their - // implementation code. - if (entry.EntryType == 1 && entry.EndAddress == Conv64BrokenEndAddress) - return physicalBytesAvailable; - - // T64 stores a 16-bit exclusive end address. An entry that ends exactly at - // $10000 therefore wraps the stored field to zero; this is the only valid - // wrap and is needed for payloads reaching the top of C64 memory. - var logicalEnd = entry.EndAddress == 0 && entry.StartAddress != 0 - ? 0x10000L - : entry.EndAddress; - if (logicalEnd < entry.StartAddress) - throw new InvalidDataException($"T64: entry {entry.DirectoryIndex} has an end address before its start address."); - - return logicalEnd - entry.StartAddress; - } - - private static bool IsKnownSignature(string signature) - => signature.StartsWith("C64S tape image file", StringComparison.Ordinal) - || signature.StartsWith("C64 tape image file", StringComparison.Ordinal) - || signature.StartsWith("C64S tape file", StringComparison.Ordinal) - || signature.StartsWith("C64S image file", StringComparison.Ordinal); - /// /// Decodes the supplied input. /// public byte[] Extract(T64Entry entry) { ArgumentNullException.ThrowIfNull(entry); if (entry.Size == 0) return []; - if (entry.DataOffset < 0 || entry.Size > int.MaxValue || (long)entry.DataOffset + entry.Size > _data.Length) - throw new InvalidDataException("T64: entry payload is outside the image."); - return _data.AsSpan(entry.DataOffset, (int)entry.Size).ToArray(); + var offset = entry.DataOffset; + var length = (int)entry.Size; + if (offset + length > _data.Length) + length = Math.Max(0, _data.Length - offset); + return _data.AsSpan(offset, length).ToArray(); } /// /// Releases resources held by this instance. /// public void Dispose() { } - - private readonly record struct RawEntry( - int DirectoryIndex, - byte EntryType, - byte FileType, - ushort StartAddress, - ushort EndAddress, - int DataOffset, - string Name); } diff --git a/FileFormats/FileFormat.T64/T64Writer.cs b/FileFormats/FileFormat.T64/T64Writer.cs index 67a7d0f9b..4fbe970a1 100644 --- a/FileFormats/FileFormat.T64/T64Writer.cs +++ b/FileFormats/FileFormat.T64/T64Writer.cs @@ -8,89 +8,75 @@ namespace FileFormat.T64; /// Writes a Commodore 64 T64 tape container, building the tape record and the directory of program entries. /// public sealed class T64Writer { - private readonly List<(string Name, ushort StartAddress, byte FileType, byte[] Data)> _files = []; + private readonly List<(string Name, ushort StartAddress, byte[] Data)> _files = []; /// /// Performs the add file operation. /// - public void AddFile(string name, ushort startAddress, byte[] data) - => AddFile(name, startAddress, 0x82, data); - - /// - /// Adds a file while preserving its Commodore file-type byte. - /// - public void AddFile(string name, ushort startAddress, byte fileType, byte[] data) { - ArgumentNullException.ThrowIfNull(name); - ArgumentNullException.ThrowIfNull(data); - _files.Add((name, startAddress, fileType, data)); - } + public void AddFile(string name, ushort startAddress, byte[] data) => + _files.Add((name, startAddress, data)); /// /// Performs the add file operation. /// - public void AddFile(string name, byte[] data) - => AddFile(name, 0x0801, 0x82, data); // default BASIC start address + PRG type + public void AddFile(string name, byte[] data) => + _files.Add((name, 0x0801, data)); // default BASIC start address /// /// Performs the build operation. /// - public byte[] Build(string tapeName = "TAPE", ushort version = 0x0100) { - ArgumentNullException.ThrowIfNull(tapeName); - if (_files.Count > ushort.MaxValue) - throw new InvalidOperationException("T64: directory entry count exceeds the 16-bit header field."); - - const int headerSize = 64; - const int entrySize = 32; - var dirSize = checked(_files.Count * entrySize); + public byte[] Build(string tapeName = "TAPE") { + var headerSize = 64; + var dirSize = _files.Count * 32; var dataStart = headerSize + dirSize; - // A tape entry addresses its file inside the C64's 64 KB memory map, so no - // entry can be larger than that. The archive itself is currently built in - // memory, therefore it also has to fit a managed byte array. + // entry can be larger than that — and the archive is built in memory. + // Summing the sizes in an int overflowed before either limit was reported. var totalBytes = (long)dataStart; - foreach (var (name, startAddr, _, data) in _files) { + foreach (var (name, startAddr, data) in _files) { if (startAddr + (long)data.Length > 0x10000) throw new InvalidOperationException( $"T64: '{name}' is {data.Length:N0} bytes and loads at ${startAddr:X4}, past the end of " + "the C64's 64 KB address space that a tape entry's start/end addresses describe."); totalBytes += data.Length; - if (totalBytes > Array.MaxLength) - throw new InvalidOperationException("T64: image is too large for the in-memory writer."); } - var totalSize = checked((int)totalBytes); + var totalSize = (int)totalBytes; var output = new byte[totalSize]; // Header var sig = "C64S tape image file\0\0\0\0\0\0\0\0\0\0\0\0"u8; sig[..32].CopyTo(output); - BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(32), version); - BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(34), (ushort)_files.Count); - BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(36), (ushort)_files.Count); + BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(32), 0x0100); // version + BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(34), (ushort)_files.Count); // max entries + BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(36), (ushort)_files.Count); // used entries // Tape name (24 bytes at offset 40) - var tapeNameBytes = Encoding.ASCII.GetBytes(tapeName.Length > 24 ? tapeName[..24] : tapeName); - tapeNameBytes.CopyTo(output, 40); - output.AsSpan(40 + tapeNameBytes.Length, 24 - tapeNameBytes.Length).Fill(0x20); + var nameBytes = Encoding.ASCII.GetBytes(tapeName.Length > 24 ? tapeName[..24] : tapeName); + nameBytes.CopyTo(output, 40); + for (var j = nameBytes.Length; j < 24; j++) + output[40 + j] = 0x20; // Directory entries + data var dataOffset = dataStart; for (var i = 0; i < _files.Count; i++) { - var (name, startAddr, fileType, data) = _files[i]; - var entryOff = headerSize + i * entrySize; + var (name, startAddr, data) = _files[i]; + var entryOff = headerSize + i * 32; output[entryOff] = 1; // normal entry - output[entryOff + 1] = fileType; + output[entryOff + 1] = 0x82; // PRG BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(entryOff + 2), startAddr); - BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(entryOff + 4), unchecked((ushort)(startAddr + data.Length))); + BinaryPrimitives.WriteUInt16LittleEndian(output.AsSpan(entryOff + 4), (ushort)(startAddr + data.Length)); BinaryPrimitives.WriteUInt32LittleEndian(output.AsSpan(entryOff + 8), (uint)dataOffset); // Filename var fnameBytes = Encoding.ASCII.GetBytes(name.Length > 16 ? name[..16] : name); fnameBytes.CopyTo(output, entryOff + 16); - output.AsSpan(entryOff + 16 + fnameBytes.Length, 16 - fnameBytes.Length).Fill(0x20); + for (var j = fnameBytes.Length; j < 16; j++) + output[entryOff + 16 + j] = 0x20; + // Copy data data.CopyTo(output, dataOffset); dataOffset += data.Length; } From 4d202ef6f26743e5d75234a9228416fc9ac12754 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 13:21:06 +0200 Subject: [PATCH 163/165] - drop the NRG and T64 maintenance branches from this round Both fail their own tests once integrated, in ways that need a format decision rather than a merge repair: - NRG writes CUEX entry 1 (track 1, index 0) at LBA -150 where its own fixture expects 0. One of the two states the Red Book pregap convention wrongly, and which is a question about the format, not about the merge. - T64 reads back "entry 0 extends into the next payload or past EOF" on the wild-image fixture the same branch adds. They return to their own pull requests; the other fourteen land here. --- Compression.Tests/Nrg/NrgAuthoringTests.cs | 269 --------- .../Nrg/NrgInPlaceModifyTests.cs | 540 ++++++++++++------ Compression.Tests/Nrg/NrgTests.cs | 397 +++++-------- .../FileFormat.Nrg/NrgFormatDescriptor.cs | 252 +++----- .../FileFormat.Nrg/NrgInPlaceModifier.cs | 356 +++++++----- FileFormats/FileFormat.Nrg/NrgReader.cs | 438 ++++++-------- .../FileFormat.Nrg/NrgStructureInspector.cs | 131 ----- FileFormats/FileFormat.Nrg/NrgWriter.cs | 365 ------------ 8 files changed, 982 insertions(+), 1766 deletions(-) delete mode 100644 Compression.Tests/Nrg/NrgAuthoringTests.cs delete mode 100644 FileFormats/FileFormat.Nrg/NrgStructureInspector.cs delete mode 100644 FileFormats/FileFormat.Nrg/NrgWriter.cs diff --git a/Compression.Tests/Nrg/NrgAuthoringTests.cs b/Compression.Tests/Nrg/NrgAuthoringTests.cs deleted file mode 100644 index b54d76fad..000000000 --- a/Compression.Tests/Nrg/NrgAuthoringTests.cs +++ /dev/null @@ -1,269 +0,0 @@ -using System.Buffers.Binary; -using Compression.Registry; -using FileFormat.Nrg; - -namespace Compression.Tests.Nrg; - -[TestFixture] -public class NrgAuthoringTests { - private const int CookedSectorSize = 2048; - private const int AudioSectorSize = 2352; - - [Test, Category("RoundTrip")] - public void Write_MixedModeDao_PreservesAudioAndFindsDataTrack() { - var audio = Pattern(AudioSectorSize * 3, 17); - var iso = BuildIso("SECOND.BIN", "second-track"u8.ToArray()); - var disc = new NrgDiscDefinition([ - new NrgSessionDefinition([ - new NrgTrackDefinition(NrgTrackMode.Audio, audio) { - PregapSectors = 150, - Isrc = "USABC2600001", - }, - new NrgTrackDefinition(NrgTrackMode.Mode1, iso), - ]) { Mcn = "1234567890123" }, - ]); - - var image = Write(disc); - var chunks = ReadChunks(image); - Assert.That(chunks.Select(static chunk => chunk.Id), Is.EqualTo(new[] { "CUEX", "DAOX", "SINF", "MTYP", "END!" })); - - var cuex = chunks.Single(static chunk => chunk.Id == "CUEX").Payload; - Assert.That(cuex.Length, Is.EqualTo(48), "lead-in + two indices per track + lead-out"); - AssertCue(cuex, 0, adrCtl: 0x01, track: 0x00, index: 0x00, lba: -150); - AssertCue(cuex, 1, adrCtl: 0x01, track: 0x01, index: 0x00, lba: 0); - AssertCue(cuex, 2, adrCtl: 0x01, track: 0x01, index: 0x01, lba: 150); - AssertCue(cuex, 3, adrCtl: 0x41, track: 0x02, index: 0x00, lba: 153); - AssertCue(cuex, 4, adrCtl: 0x41, track: 0x02, index: 0x01, lba: 153); - AssertCue(cuex, 5, adrCtl: 0x41, track: 0xAA, index: 0x01, lba: 153 + iso.Length / CookedSectorSize); - - var daox = chunks.Single(static chunk => chunk.Id == "DAOX").Payload; - Assert.That(daox.Length, Is.EqualTo(22 + 2 * 42)); - Assert.Multiple(() => { - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(daox), Is.EqualTo((uint)daox.Length)); - Assert.That(System.Text.Encoding.ASCII.GetString(daox, 4, 13), Is.EqualTo("1234567890123")); - Assert.That(daox[20], Is.EqualTo(1)); - Assert.That(daox[21], Is.EqualTo(2)); - }); - - var audioRecord = daox.AsSpan(22, 42); - var dataRecord = daox.AsSpan(64, 42); - var audioPregap = checked((long)BinaryPrimitives.ReadUInt64BigEndian(audioRecord[18..])); - var audioStart = checked((long)BinaryPrimitives.ReadUInt64BigEndian(audioRecord[26..])); - var audioEnd = checked((long)BinaryPrimitives.ReadUInt64BigEndian(audioRecord[34..])); - var dataStart = checked((long)BinaryPrimitives.ReadUInt64BigEndian(dataRecord[26..])); - var dataEnd = checked((long)BinaryPrimitives.ReadUInt64BigEndian(dataRecord[34..])); - - // Materialised before the closure: a lambda cannot capture a ref struct. - var audioBytes = audioRecord.ToArray(); - var dataBytes = dataRecord.ToArray(); - Assert.Multiple(() => { - Assert.That(BinaryPrimitives.ReadUInt16BigEndian(audioBytes.AsSpan(12)), Is.EqualTo(AudioSectorSize)); - Assert.That(audioBytes[14], Is.EqualTo((byte)NrgTrackMode.Audio)); - Assert.That(System.Text.Encoding.ASCII.GetString(audioBytes.AsSpan(0, 12)), Is.EqualTo("USABC2600001")); - Assert.That(audioPregap, Is.Zero); - Assert.That(audioStart - audioPregap, Is.EqualTo(150L * AudioSectorSize)); - Assert.That(audioEnd - audioStart, Is.EqualTo(audio.Length)); - Assert.That(BinaryPrimitives.ReadUInt16BigEndian(dataBytes.AsSpan(12)), Is.EqualTo(CookedSectorSize)); - Assert.That(dataBytes[14], Is.EqualTo((byte)NrgTrackMode.Mode1)); - Assert.That(dataStart, Is.EqualTo(audioEnd)); - Assert.That(dataEnd - dataStart, Is.EqualTo(iso.Length)); - }); - - Assert.That(image.AsSpan(checked((int)audioStart), audio.Length).ToArray(), Is.EqualTo(audio)); - Assert.That(image.AsSpan(checked((int)dataStart + 16 * CookedSectorSize), 6).ToArray(), - Is.EqualTo(new byte[] { 1, (byte)'C', (byte)'D', (byte)'0', (byte)'0', (byte)'1' })); - - using var stream = new MemoryStream(image, writable: false); - using var reader = new NrgReader(stream); - var entry = reader.Entries.Single(e => !e.IsDirectory && e.Name.Equals("SECOND.BIN", StringComparison.OrdinalIgnoreCase)); - Assert.That(reader.Extract(entry), Is.EqualTo("second-track"u8.ToArray())); - } - - [Test, Category("RoundTrip")] - public void Write_MultipleSessions_UsesCdLeadOutAndLeadInSpacing() { - var firstIso = BuildIso("FIRST.BIN", "first"u8.ToArray()); - var secondIso = BuildIso("SECOND.BIN", "second"u8.ToArray()); - var image = Write(new NrgDiscDefinition([ - new NrgSessionDefinition([ - new NrgTrackDefinition(NrgTrackMode.Mode1, firstIso), - ]), - new NrgSessionDefinition([ - new NrgTrackDefinition(NrgTrackMode.Mode1, secondIso), - ]), - ])); - - var chunks = ReadChunks(image); - Assert.That(chunks.Select(static chunk => chunk.Id), Is.EqualTo(new[] { - "CUEX", "DAOX", "SINF", "CUEX", "DAOX", "SINF", "MTYP", "END!", - })); - - var cueSessions = chunks.Where(static chunk => chunk.Id == "CUEX").Select(static chunk => chunk.Payload).ToArray(); - var firstLeadOut = CueLba(cueSessions[0], 3); - var secondLeadIn = CueLba(cueSessions[1], 0); - var secondTrackIndex0 = CueLba(cueSessions[1], 1); - var secondTrackIndex1 = CueLba(cueSessions[1], 2); - - Assert.Multiple(() => { - Assert.That(secondLeadIn, Is.EqualTo(firstLeadOut + 6750), "first session has the long CD lead-out"); - Assert.That(secondTrackIndex0, Is.EqualTo(secondLeadIn + 4500), "next session starts after its lead-in"); - Assert.That(secondTrackIndex1, Is.EqualTo(secondTrackIndex0)); - Assert.That(cueSessions[0][9], Is.EqualTo(0x01), "first session uses track 1 BCD"); - Assert.That(cueSessions[1][9], Is.EqualTo(0x02), "track numbering remains global across sessions"); - }); - - var sinf = chunks.Where(static chunk => chunk.Id == "SINF").ToArray(); - Assert.That(sinf, Has.Length.EqualTo(2)); - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(sinf[0].Payload), Is.EqualTo(1)); - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(sinf[1].Payload), Is.EqualTo(1)); - } - - [Test, Category("RoundTrip")] - public void Write_CdText_WritesRaw18BytePacks() { - var cdText = Pattern(36, 91); - var image = Write(new NrgDiscDefinition([ - new NrgSessionDefinition([ - new NrgTrackDefinition(NrgTrackMode.Audio, Pattern(AudioSectorSize, 33)), - ]), - ]) { CdText = cdText }); - - var chunks = ReadChunks(image); - Assert.That(chunks.Select(static chunk => chunk.Id), Is.EqualTo(new[] { "CUEX", "DAOX", "SINF", "CDTX", "MTYP", "END!" })); - Assert.That(chunks.Single(static chunk => chunk.Id == "CDTX").Payload, Is.EqualTo(cdText)); - } - - [Test, Category("Boundary")] - public void Write_RejectsMisalignedAudioTrack() { - using var output = new MemoryStream(); - var disc = new NrgDiscDefinition([ - new NrgSessionDefinition([ - new NrgTrackDefinition(NrgTrackMode.Audio, new byte[AudioSectorSize + 1]), - ]), - ]); - - Assert.Throws(() => NrgWriter.Write(output, disc)); - } - - [Test, Category("Boundary")] - public void Write_RejectsMoreThan99Tracks() { - var tracks = Enumerable.Range(0, 100) - .Select(static _ => new NrgTrackDefinition(NrgTrackMode.Audio, new byte[AudioSectorSize])) - .ToArray(); - using var output = new MemoryStream(); - Assert.Throws(() => NrgWriter.Write(output, new NrgDiscDefinition([ - new NrgSessionDefinition(tracks), - ]))); - } - - [Test, Category("Boundary")] - public void Write_RejectsInvalidMcnIsrcAndCdText() { - using var output = new MemoryStream(); - Assert.Multiple(() => { - Assert.Throws(() => NrgWriter.Write(output, new NrgDiscDefinition([ - new NrgSessionDefinition([ - new NrgTrackDefinition(NrgTrackMode.Audio, new byte[AudioSectorSize]), - ]) { Mcn = "123" }, - ]))); - - Assert.Throws(() => NrgWriter.Write(output, new NrgDiscDefinition([ - new NrgSessionDefinition([ - new NrgTrackDefinition(NrgTrackMode.Audio, new byte[AudioSectorSize]) { Isrc = "short" }, - ]), - ]))); - - Assert.Throws(() => NrgWriter.Write(output, new NrgDiscDefinition([ - new NrgSessionDefinition([ - new NrgTrackDefinition(NrgTrackMode.Audio, new byte[AudioSectorSize]), - ]), - ]) { CdText = new byte[17] })); - }); - } - - [Test, Category("Regression")] - public void MultiTrack_FileMutationAndDefragRefuseWithoutChangingImage() { - var descriptor = new NrgFormatDescriptor(); - var source = Write(new NrgDiscDefinition([ - new NrgSessionDefinition([ - new NrgTrackDefinition(NrgTrackMode.Audio, Pattern(AudioSectorSize, 4)), - new NrgTrackDefinition(NrgTrackMode.Mode1, BuildIso("KEEP.BIN", "keep"u8.ToArray())), - ]), - ])); - - using var image = new MemoryStream(); - image.Write(source); - image.Position = 0; - - Assert.Throws(() => ((IArchiveModifiable)descriptor).Add(image, [ - ArchiveInputInfo.InMemory("NOPE.BIN", "nope"u8.ToArray()), - ])); - Assert.That(image.ToArray(), Is.EqualTo(source)); - - image.Position = 0; - Assert.Throws(() => ((IArchiveDefragmentable)descriptor).Defragment(image)); - Assert.That(image.ToArray(), Is.EqualTo(source)); - } - - [Test, Category("Regression")] - public void MultiTrack_ShrinkCopiesThroughByteIdentical() { - var descriptor = new NrgFormatDescriptor(); - var source = Write(new NrgDiscDefinition([ - new NrgSessionDefinition([ - new NrgTrackDefinition(NrgTrackMode.Audio, Pattern(AudioSectorSize * 2, 19)), - new NrgTrackDefinition(NrgTrackMode.Mode1, BuildIso("KEEP.BIN", "keep"u8.ToArray())), - ]), - ])); - - using var input = new MemoryStream(source, writable: false); - using var output = new MemoryStream(); - ((IArchiveShrinkable)descriptor).Shrink(input, output); - Assert.That(output.ToArray(), Is.EqualTo(source)); - } - - private static byte[] BuildIso(string name, byte[] data) { - var writer = new FileSystem.Iso.IsoWriter(); - writer.AddFile(name, data); - return writer.Build(); - } - - private static byte[] Write(NrgDiscDefinition disc) { - using var output = new MemoryStream(); - NrgWriter.Write(output, disc); - return output.ToArray(); - } - - private static byte[] Pattern(int length, int multiplier) - => Enumerable.Range(0, length).Select(i => unchecked((byte)(i * multiplier + 3))).ToArray(); - - private sealed record Chunk(string Id, byte[] Payload); - - private static List ReadChunks(byte[] image) { - Assert.That(image.AsSpan(image.Length - 12, 4).ToArray(), Is.EqualTo("NER5"u8.ToArray())); - var position = checked((int)BinaryPrimitives.ReadUInt64BigEndian(image.AsSpan(image.Length - 8))); - var footerOffset = image.Length - 12; - var chunks = new List(); - while (position <= footerOffset - 8) { - var id = System.Text.Encoding.ASCII.GetString(image, position, 4); - var length = checked((int)BinaryPrimitives.ReadUInt32BigEndian(image.AsSpan(position + 4, 4))); - Assert.That(position + 8L + length, Is.LessThanOrEqualTo(footerOffset)); - var payload = image.AsSpan(position + 8, length).ToArray(); - chunks.Add(new Chunk(id, payload)); - position += 8 + length; - if (id == "END!") - break; - } - Assert.That(position, Is.EqualTo(footerOffset)); - return chunks; - } - - private static void AssertCue(byte[] payload, int entryIndex, byte adrCtl, byte track, byte index, int lba) { - var offset = entryIndex * 8; - Assert.Multiple(() => { - Assert.That(payload[offset], Is.EqualTo(adrCtl)); - Assert.That(payload[offset + 1], Is.EqualTo(track)); - Assert.That(payload[offset + 2], Is.EqualTo(index)); - Assert.That(CueLba(payload, entryIndex), Is.EqualTo(lba)); - }); - } - - private static int CueLba(byte[] payload, int entryIndex) - => unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(payload.AsSpan(entryIndex * 8 + 4, 4))); -} diff --git a/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs b/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs index 7ebf0ee52..1818877dc 100644 --- a/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs +++ b/Compression.Tests/Nrg/NrgInPlaceModifyTests.cs @@ -1,5 +1,4 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; using Compression.Registry; using FileFormat.Nrg; @@ -7,139 +6,172 @@ namespace Compression.Tests.Nrg; [TestFixture] public class NrgInPlaceModifyTests { + private const int Iso9660SectorSize = 2048; private const int RawSectorSize = 2352; private const int Mode1DataOffset = 16; private const int Ner5FooterSize = 12; private const int NeroFooterSize = 8; + // ── geometry fixtures ─────────────────────────────────────────────── + + /// + /// Builds a raw 2 352-byte/sector Mode 1 NRG v2 image: PVD at LBA 16 with + /// CD001 + 12-byte "NER5" footer at EOF. + /// private static byte[] BuildRawMode1Nrg(int sectorCount = 32) { - if (sectorCount <= 16) - sectorCount = 20; - var dataLength = sectorCount * RawSectorSize; - var image = new byte[dataLength + Ner5FooterSize]; - - for (var i = 0; i < dataLength; ++i) - image[i] = (byte)((i * 31 + 7) & 0xFF); - - ReadOnlySpan sync = [ - 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, - ]; - for (var sector = 0; sector < sectorCount; ++sector) { - sync.CopyTo(image.AsSpan(sector * RawSectorSize, 12)); - image[sector * RawSectorSize + 12] = 0; - image[sector * RawSectorSize + 13] = 0; - image[sector * RawSectorSize + 14] = (byte)sector; - image[sector * RawSectorSize + 15] = 0x01; + if (sectorCount <= 16) sectorCount = 20; + var dataLen = sectorCount * RawSectorSize; + var buf = new byte[dataLen + Ner5FooterSize]; + + for (var i = 0; i < dataLen; i++) + buf[i] = (byte)((i * 31 + 7) & 0xFF); + + Span sync = stackalloc byte[12] { + 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, + }; + for (var s = 0; s < sectorCount; s++) { + sync.CopyTo(buf.AsSpan(s * RawSectorSize, 12)); + buf[s * RawSectorSize + 12] = 0; + buf[s * RawSectorSize + 13] = 0; + buf[s * RawSectorSize + 14] = (byte)s; + buf[s * RawSectorSize + 15] = 0x01; } var pvdAt = 16 * RawSectorSize + Mode1DataOffset; - image[pvdAt] = 1; - "CD001"u8.CopyTo(image.AsSpan(pvdAt + 1)); - image[pvdAt + 6] = 1; - - "NER5"u8.CopyTo(image.AsSpan(dataLength)); - BinaryPrimitives.WriteUInt64BigEndian(image.AsSpan(dataLength + 4), checked((ulong)dataLength)); - return image; + buf[pvdAt + 0] = 1; + buf[pvdAt + 1] = (byte)'C'; + buf[pvdAt + 2] = (byte)'D'; + buf[pvdAt + 3] = (byte)'0'; + buf[pvdAt + 4] = (byte)'0'; + buf[pvdAt + 5] = (byte)'1'; + buf[pvdAt + 6] = 1; + + // NER5 footer: "NER5" + uint64 BE chunk-table offset (points at end of data). + buf[dataLen + 0] = (byte)'N'; + buf[dataLen + 1] = (byte)'E'; + buf[dataLen + 2] = (byte)'R'; + buf[dataLen + 3] = (byte)'5'; + System.Buffers.Binary.BinaryPrimitives.WriteUInt64BigEndian( + buf.AsSpan(dataLen + 4, 8), (ulong)dataLen); + + return buf; } + /// + /// Builds a cooked 2 048-byte/sector NRG v2 image. + /// private static byte[] BuildCookedNrg(int sectorCount = 32) { - if (sectorCount <= 16) - sectorCount = 20; - var dataLength = sectorCount * Iso9660SectorSize; - var image = new byte[dataLength + Ner5FooterSize]; + if (sectorCount <= 16) sectorCount = 20; + var dataLen = sectorCount * Iso9660SectorSize; + var buf = new byte[dataLen + Ner5FooterSize]; - for (var i = 0; i < dataLength; ++i) - image[i] = (byte)((i * 17 + 3) & 0xFF); + for (var i = 0; i < dataLen; i++) + buf[i] = (byte)((i * 17 + 3) & 0xFF); var pvdAt = 16 * Iso9660SectorSize; - image[pvdAt] = 1; - "CD001"u8.CopyTo(image.AsSpan(pvdAt + 1)); - image[pvdAt + 6] = 1; - - "NER5"u8.CopyTo(image.AsSpan(dataLength)); - BinaryPrimitives.WriteUInt64BigEndian(image.AsSpan(dataLength + 4), checked((ulong)dataLength)); - return image; + buf[pvdAt + 0] = 1; + buf[pvdAt + 1] = (byte)'C'; + buf[pvdAt + 2] = (byte)'D'; + buf[pvdAt + 3] = (byte)'0'; + buf[pvdAt + 4] = (byte)'0'; + buf[pvdAt + 5] = (byte)'1'; + buf[pvdAt + 6] = 1; + + buf[dataLen + 0] = (byte)'N'; + buf[dataLen + 1] = (byte)'E'; + buf[dataLen + 2] = (byte)'R'; + buf[dataLen + 3] = (byte)'5'; + System.Buffers.Binary.BinaryPrimitives.WriteUInt64BigEndian( + buf.AsSpan(dataLen + 4, 8), (ulong)dataLen); + return buf; } + /// + /// Builds a raw NRG v1 image: 8-byte "NERO" footer at EOF. + /// private static byte[] BuildRawMode1NrgV1(int sectorCount = 32) { - if (sectorCount <= 16) - sectorCount = 20; - var dataLength = sectorCount * RawSectorSize; - var image = new byte[dataLength + NeroFooterSize]; - - for (var i = 0; i < dataLength; ++i) - image[i] = (byte)((i * 31 + 7) & 0xFF); - - ReadOnlySpan sync = [ - 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, - ]; - for (var sector = 0; sector < sectorCount; ++sector) { - sync.CopyTo(image.AsSpan(sector * RawSectorSize, 12)); - image[sector * RawSectorSize + 14] = (byte)sector; - image[sector * RawSectorSize + 15] = 0x01; + if (sectorCount <= 16) sectorCount = 20; + var dataLen = sectorCount * RawSectorSize; + var buf = new byte[dataLen + NeroFooterSize]; + + for (var i = 0; i < dataLen; i++) + buf[i] = (byte)((i * 31 + 7) & 0xFF); + + Span sync = stackalloc byte[12] { + 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, + }; + for (var s = 0; s < sectorCount; s++) { + sync.CopyTo(buf.AsSpan(s * RawSectorSize, 12)); + buf[s * RawSectorSize + 14] = (byte)s; + buf[s * RawSectorSize + 15] = 0x01; } var pvdAt = 16 * RawSectorSize + Mode1DataOffset; - image[pvdAt] = 1; - "CD001"u8.CopyTo(image.AsSpan(pvdAt + 1)); - image[pvdAt + 6] = 1; - - "NERO"u8.CopyTo(image.AsSpan(dataLength)); - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(dataLength + 4), checked((uint)dataLength)); - return image; + buf[pvdAt + 0] = 1; + buf[pvdAt + 1] = (byte)'C'; + buf[pvdAt + 2] = (byte)'D'; + buf[pvdAt + 3] = (byte)'0'; + buf[pvdAt + 4] = (byte)'0'; + buf[pvdAt + 5] = (byte)'1'; + buf[pvdAt + 6] = 1; + + buf[dataLen + 0] = (byte)'N'; + buf[dataLen + 1] = (byte)'E'; + buf[dataLen + 2] = (byte)'R'; + buf[dataLen + 3] = (byte)'O'; + System.Buffers.Binary.BinaryPrimitives.WriteUInt32BigEndian( + buf.AsSpan(dataLen + 4, 4), (uint)dataLen); + return buf; } - private static byte[] BuildRealNrg() { - var descriptor = new NrgFormatDescriptor(); - using var image = new MemoryStream(); - ((IArchiveCreatable)descriptor).Create( - image, - [ArchiveInputInfo.InMemory("DATA.BIN", Enumerable.Range(0, 4096).Select(static i => (byte)i).ToArray())], - new FormatCreateOptions()); - return image.ToArray(); - } - - private static int TrailerOffset(byte[] image) - => checked((int)BinaryPrimitives.ReadUInt64BigEndian(image.AsSpan(image.Length - 8))); + // ── WriteSector — happy path ───────────────────────────────────────── [Test, Category("RoundTrip")] public void WriteSector_Raw_RewritesUserDataOnly_FramingPreserved() { var image = BuildRawMode1Nrg(); var original = (byte[])image.Clone(); - using var stream = new MemoryStream(image, writable: true); - var payload = Enumerable.Range(0, Iso9660SectorSize).Select(static i => (byte)(i + 1)).ToArray(); + using var ms = new MemoryStream(image, writable: true); + + var payload = new byte[Iso9660SectorSize]; + for (var i = 0; i < payload.Length; i++) payload[i] = (byte)((i + 1) & 0xFF); - NrgInPlaceModifier.WriteSector(stream, 20, payload); + NrgInPlaceModifier.WriteSector(ms, lba: 20, payload); var userDataAt = 20 * RawSectorSize + Mode1DataOffset; Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); - Assert.That(image.AsSpan(20 * RawSectorSize, Mode1DataOffset).ToArray(), - Is.EqualTo(original.AsSpan(20 * RawSectorSize, Mode1DataOffset).ToArray())); - Assert.That(image.AsSpan(userDataAt + Iso9660SectorSize, RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray(), - Is.EqualTo(original.AsSpan(userDataAt + Iso9660SectorSize, RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray())); - Assert.That(image.AsSpan(image.Length - Ner5FooterSize).ToArray(), - Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize).ToArray())); + + Assert.That(image.AsSpan(20 * RawSectorSize, 16).ToArray(), + Is.EqualTo(original.AsSpan(20 * RawSectorSize, 16).ToArray())); + + Assert.That(image.AsSpan(userDataAt + Iso9660SectorSize, + RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray(), + Is.EqualTo(original.AsSpan(userDataAt + Iso9660SectorSize, + RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray())); + + // NER5 footer byte-identical. + Assert.That(image.AsSpan(image.Length - Ner5FooterSize, Ner5FooterSize).ToArray(), + Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize, Ner5FooterSize).ToArray())); } [Test, Category("RoundTrip")] public void WriteSector_Raw_OtherSectors_ByteIdentical() { var image = BuildRawMode1Nrg(); var original = (byte[])image.Clone(); - using var stream = new MemoryStream(image, writable: true); - var payload = Enumerable.Repeat((byte)0xA5, Iso9660SectorSize).ToArray(); + using var ms = new MemoryStream(image, writable: true); - NrgInPlaceModifier.WriteSector(stream, 20, payload); + var payload = new byte[Iso9660SectorSize]; + Array.Fill(payload, (byte)0xA5); + + NrgInPlaceModifier.WriteSector(ms, lba: 20, payload); var sectorCount = (image.Length - Ner5FooterSize) / RawSectorSize; - for (var lba = 0; lba < sectorCount; ++lba) { - if (lba == 20) - continue; - var offset = lba * RawSectorSize; - Assert.That(image.AsSpan(offset, RawSectorSize).ToArray(), - Is.EqualTo(original.AsSpan(offset, RawSectorSize).ToArray()), $"LBA {lba} unexpectedly changed."); + for (var lba = 0; lba < sectorCount; lba++) { + if (lba == 20) continue; + var sectorOff = lba * RawSectorSize; + Assert.That(image.AsSpan(sectorOff, RawSectorSize).ToArray(), + Is.EqualTo(original.AsSpan(sectorOff, RawSectorSize).ToArray()), + $"LBA {lba} unexpectedly changed."); } } @@ -147,150 +179,277 @@ public void WriteSector_Raw_OtherSectors_ByteIdentical() { public void WriteSector_Cooked_RewritesUserDataAtSectorOffset() { var image = BuildCookedNrg(); var original = (byte[])image.Clone(); - using var stream = new MemoryStream(image, writable: true); - var payload = Enumerable.Range(0, Iso9660SectorSize).Select(static i => (byte)i).ToArray(); + using var ms = new MemoryStream(image, writable: true); + + var payload = new byte[Iso9660SectorSize]; + for (var i = 0; i < payload.Length; i++) payload[i] = (byte)(i & 0xFF); - NrgInPlaceModifier.WriteSector(stream, 22, payload); + NrgInPlaceModifier.WriteSector(ms, lba: 22, payload); var sectorAt = 22 * Iso9660SectorSize; Assert.That(image.AsSpan(sectorAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); - Assert.That(image.AsSpan(0, sectorAt).ToArray(), Is.EqualTo(original.AsSpan(0, sectorAt).ToArray())); + + Assert.That(image.AsSpan(0, sectorAt).ToArray(), + Is.EqualTo(original.AsSpan(0, sectorAt).ToArray())); Assert.That(image.AsSpan(sectorAt + Iso9660SectorSize).ToArray(), Is.EqualTo(original.AsSpan(sectorAt + Iso9660SectorSize).ToArray())); } - [Test, Category("Regression")] - public void WriteSector_PastTrackEnd_RefusesInsteadOfCorruptingChunkOffsets() { + // ── WriteSector — append path ──────────────────────────────────────── + + [Test, Category("RoundTrip")] + public void WriteSector_PastEof_AppendsWithCorrectFraming_FooterRelocated() { var image = BuildRawMode1Nrg(sectorCount: 20); var original = (byte[])image.Clone(); - using var stream = new MemoryStream(); - stream.Write(image); - stream.Position = 0; - - Assert.That( - () => NrgInPlaceModifier.WriteSector(stream, 25, new byte[Iso9660SectorSize]), - Throws.TypeOf()); - Assert.That(stream.ToArray(), Is.EqualTo(original)); + using var ms = new MemoryStream(); + ms.Write(image); + + var payload = new byte[Iso9660SectorSize]; + Array.Fill(payload, (byte)0x5A); + + NrgInPlaceModifier.WriteSector(ms, lba: 25, payload); + + var grown = ms.ToArray(); + Assert.That(grown.Length, Is.EqualTo(26 * RawSectorSize + Ner5FooterSize)); + + Assert.That(grown.AsSpan(0, 20 * RawSectorSize).ToArray(), + Is.EqualTo(original.AsSpan(0, 20 * RawSectorSize).ToArray())); + + var userDataAt = 25 * RawSectorSize + Mode1DataOffset; + Assert.That(grown.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); + + for (var lba = 20; lba <= 24; lba++) { + var sectorOff = lba * RawSectorSize; + Assert.That(grown[sectorOff + 0], Is.EqualTo(0x00)); + Assert.That(grown[sectorOff + 1], Is.EqualTo(0xFF)); + Assert.That(grown[sectorOff + 15], Is.EqualTo(0x01), $"LBA {lba} mode byte"); + } + + Assert.That(grown.AsSpan(grown.Length - Ner5FooterSize, Ner5FooterSize).ToArray(), + Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize, Ner5FooterSize).ToArray())); } + // ── ZeroSector ────────────────────────────────────────────────────── + [Test, Category("RoundTrip")] public void ZeroSector_WipesUserData_PreservesFramingAndFooter() { var image = BuildRawMode1Nrg(); var original = (byte[])image.Clone(); - using var stream = new MemoryStream(image, writable: true); + using var ms = new MemoryStream(image, writable: true); - Assert.That(NrgInPlaceModifier.ZeroSector(stream, 19), Is.True); + Assert.That(NrgInPlaceModifier.ZeroSector(ms, lba: 19), Is.True); var userDataAt = 19 * RawSectorSize + Mode1DataOffset; - Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(new byte[Iso9660SectorSize])); - Assert.That(image.AsSpan(19 * RawSectorSize, Mode1DataOffset).ToArray(), - Is.EqualTo(original.AsSpan(19 * RawSectorSize, Mode1DataOffset).ToArray())); - Assert.That(image.AsSpan(userDataAt + Iso9660SectorSize, RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray(), - Is.EqualTo(original.AsSpan(userDataAt + Iso9660SectorSize, RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray())); - Assert.That(image.AsSpan(image.Length - Ner5FooterSize).ToArray(), - Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize).ToArray())); + Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), + Is.EqualTo(new byte[Iso9660SectorSize])); + + Assert.That(image.AsSpan(19 * RawSectorSize, 16).ToArray(), + Is.EqualTo(original.AsSpan(19 * RawSectorSize, 16).ToArray())); + + Assert.That(image.AsSpan(userDataAt + Iso9660SectorSize, + RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray(), + Is.EqualTo(original.AsSpan(userDataAt + Iso9660SectorSize, + RawSectorSize - Mode1DataOffset - Iso9660SectorSize).ToArray())); + + Assert.That(image.AsSpan(image.Length - Ner5FooterSize, Ner5FooterSize).ToArray(), + Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize, Ner5FooterSize).ToArray())); } [Test, Category("RoundTrip")] public void ZeroSector_PastEof_ReturnsFalse() { var image = BuildRawMode1Nrg(); - using var stream = new MemoryStream(image, writable: true); - Assert.That(NrgInPlaceModifier.ZeroSector(stream, 9999), Is.False); + using var ms = new MemoryStream(image, writable: true); + Assert.That(NrgInPlaceModifier.ZeroSector(ms, lba: 9999), Is.False); } + // ── NRG v1 (NERO) footer ─────────────────────────────────────────── + [Test, Category("RoundTrip")] public void DetectGeometry_StripsNeroV1Footer_FromDataAreaLength() { - var image = BuildRawMode1NrgV1(); - using var stream = new MemoryStream(image, writable: true); - var geometry = NrgInPlaceModifier.DetectGeometry(stream); - - Assert.Multiple(() => { - Assert.That(geometry.SectorSize, Is.EqualTo(RawSectorSize)); - Assert.That(geometry.DataOffset, Is.EqualTo(Mode1DataOffset)); - Assert.That(geometry.DataAreaLength, Is.EqualTo(32 * RawSectorSize)); - }); + var image = BuildRawMode1NrgV1(sectorCount: 32); + using var ms = new MemoryStream(image, writable: true); + var geom = NrgInPlaceModifier.DetectGeometry(ms); + Assert.That(geom.SectorSize, Is.EqualTo(RawSectorSize)); + Assert.That(geom.DataOffset, Is.EqualTo(Mode1DataOffset)); + Assert.That(geom.DataAreaLength, Is.EqualTo(32 * RawSectorSize)); } [Test, Category("RoundTrip")] public void WriteSector_NeroV1_FooterPreserved() { var image = BuildRawMode1NrgV1(); var original = (byte[])image.Clone(); - using var stream = new MemoryStream(image, writable: true); - var payload = Enumerable.Repeat((byte)0x33, Iso9660SectorSize).ToArray(); + using var ms = new MemoryStream(image, writable: true); + + var payload = new byte[Iso9660SectorSize]; + Array.Fill(payload, (byte)0x33); + + NrgInPlaceModifier.WriteSector(ms, lba: 20, payload); + + var userDataAt = 20 * RawSectorSize + Mode1DataOffset; + Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); + + Assert.That(image.AsSpan(image.Length - NeroFooterSize, NeroFooterSize).ToArray(), + Is.EqualTo(original.AsSpan(original.Length - NeroFooterSize, NeroFooterSize).ToArray())); + } + + // ── Descriptor-level Add/Remove ───────────────────────────────────── + + [Test, Category("RoundTrip")] + public void Descriptor_Add_RewritesNamedSector() { + var image = BuildRawMode1Nrg(); + var original = (byte[])image.Clone(); + using var ms = new MemoryStream(image, writable: true); - NrgInPlaceModifier.WriteSector(stream, 20, payload); + var payload = new byte[Iso9660SectorSize]; + for (var i = 0; i < payload.Length; i++) payload[i] = (byte)(i % 251); + + ((IArchiveModifiable)new NrgFormatDescriptor()).Add(ms, [ + ArchiveInputInfo.InMemory(NrgInPlaceModifier.FormatSectorEntryName(20), payload), + ]); var userDataAt = 20 * RawSectorSize + Mode1DataOffset; Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); - Assert.That(image.AsSpan(image.Length - NeroFooterSize).ToArray(), - Is.EqualTo(original.AsSpan(original.Length - NeroFooterSize).ToArray())); + + var sectorCount = (image.Length - Ner5FooterSize) / RawSectorSize; + for (var lba = 0; lba < sectorCount; lba++) { + if (lba == 20) continue; + var off = lba * RawSectorSize; + Assert.That(image.AsSpan(off, RawSectorSize).ToArray(), + Is.EqualTo(original.AsSpan(off, RawSectorSize).ToArray()), + $"LBA {lba} unexpectedly mutated."); + } + + Assert.That(image.AsSpan(image.Length - Ner5FooterSize, Ner5FooterSize).ToArray(), + Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize, Ner5FooterSize).ToArray())); } [Test, Category("RoundTrip")] - public void MutateThenExtract_RoundTripsSectorBytes() { + public void Descriptor_Add_AppendsPastEofPreservingDataAndFooter() { + var image = BuildRawMode1Nrg(sectorCount: 20); + var original = (byte[])image.Clone(); + using var ms = new MemoryStream(); + ms.Write(image); + + var payload = new byte[Iso9660SectorSize]; + Array.Fill(payload, (byte)0x77); + + ((IArchiveModifiable)new NrgFormatDescriptor()).Add(ms, [ + ArchiveInputInfo.InMemory(NrgInPlaceModifier.FormatSectorEntryName(30), payload), + ]); + + var grown = ms.ToArray(); + Assert.That(grown.AsSpan(0, 20 * RawSectorSize).ToArray(), + Is.EqualTo(original.AsSpan(0, 20 * RawSectorSize).ToArray())); + + var userDataAt = 30 * RawSectorSize + Mode1DataOffset; + Assert.That(grown.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); + + Assert.That(grown.AsSpan(grown.Length - Ner5FooterSize, Ner5FooterSize).ToArray(), + Is.EqualTo(original.AsSpan(original.Length - Ner5FooterSize, Ner5FooterSize).ToArray())); + } + + [Test, Category("RoundTrip")] + public void Descriptor_Remove_ZerosUserData_LeavesEverythingElseIdentical() { var image = BuildRawMode1Nrg(); - using var stream = new MemoryStream(); - stream.Write(image); - var payload = Enumerable.Range(0, Iso9660SectorSize).Select(static i => (byte)(0xAA ^ i)).ToArray(); + var original = (byte[])image.Clone(); + using var ms = new MemoryStream(image, writable: true); - NrgInPlaceModifier.WriteSector(stream, 25, payload); + ((IArchiveModifiable)new NrgFormatDescriptor()).Remove(ms, [ + NrgInPlaceModifier.FormatSectorEntryName(20), + ]); - stream.Position = 25L * RawSectorSize + Mode1DataOffset; - var extracted = new byte[Iso9660SectorSize]; - stream.ReadExactly(extracted); - Assert.That(extracted, Is.EqualTo(payload)); + var userDataAt = 20 * RawSectorSize + Mode1DataOffset; + Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), + Is.EqualTo(new byte[Iso9660SectorSize])); + + Assert.That(image.AsSpan(20 * RawSectorSize, 16).ToArray(), + Is.EqualTo(original.AsSpan(20 * RawSectorSize, 16).ToArray())); + + var sectorCount = (image.Length - Ner5FooterSize) / RawSectorSize; + for (var lba = 0; lba < sectorCount; lba++) { + if (lba == 20) continue; + var off = lba * RawSectorSize; + Assert.That(image.AsSpan(off, RawSectorSize).ToArray(), + Is.EqualTo(original.AsSpan(off, RawSectorSize).ToArray()), + $"LBA {lba} unexpectedly mutated."); + } } - [Test, Category("Regression")] - public void DetectGeometry_UsesDescriptorOffset_NotFooterOffset() { - var image = BuildRealNrg(); - using var stream = new MemoryStream(image, writable: true); - var geometry = NrgInPlaceModifier.DetectGeometry(stream); - - Assert.Multiple(() => { - Assert.That(geometry.SectorSize, Is.EqualTo(Iso9660SectorSize)); - Assert.That(geometry.DataOffset, Is.Zero); - Assert.That(geometry.DataAreaLength, Is.EqualTo(TrailerOffset(image))); - Assert.That(geometry.DataAreaLength, Is.LessThan(image.Length - Ner5FooterSize), "chunk table is not sector data"); - }); + [Test, Category("RoundTrip")] + public void Descriptor_Replace_RewritesUserDataPreservingFramingAndNeighbours() { + var image = BuildRawMode1Nrg(); + var original = (byte[])image.Clone(); + using var ms = new MemoryStream(image, writable: true); + + var v1 = new byte[Iso9660SectorSize]; + Array.Fill(v1, (byte)0x11); + var v2 = new byte[Iso9660SectorSize]; + Array.Fill(v2, (byte)0x22); + + var modifier = (IArchiveModifiable)new NrgFormatDescriptor(); + modifier.Add(ms, [ArchiveInputInfo.InMemory(NrgInPlaceModifier.FormatSectorEntryName(21), v1)]); + modifier.Add(ms, [ArchiveInputInfo.InMemory(NrgInPlaceModifier.FormatSectorEntryName(21), v2)]); + + var userDataAt = 21 * RawSectorSize + Mode1DataOffset; + Assert.That(image.AsSpan(userDataAt, Iso9660SectorSize).ToArray(), Is.EqualTo(v2)); + + Assert.That(image.AsSpan(21 * RawSectorSize, 16).ToArray(), + Is.EqualTo(original.AsSpan(21 * RawSectorSize, 16).ToArray())); + + Assert.That(image.AsSpan(20 * RawSectorSize, RawSectorSize).ToArray(), + Is.EqualTo(original.AsSpan(20 * RawSectorSize, RawSectorSize).ToArray())); + Assert.That(image.AsSpan(22 * RawSectorSize, RawSectorSize).ToArray(), + Is.EqualTo(original.AsSpan(22 * RawSectorSize, RawSectorSize).ToArray())); } [Test, Category("RoundTrip")] - public void WriteSector_RealChunkedImage_PreservesWholeDescriptor() { - var image = BuildRealNrg(); - var trailer = TrailerOffset(image); - var descriptorBytes = image.AsSpan(trailer).ToArray(); - using var stream = new MemoryStream(image, writable: true); - var payload = Enumerable.Range(0, Iso9660SectorSize).Select(static i => (byte)(i * 29)).ToArray(); + public void MutateThenExtract_RoundTrip() { + var image = BuildRawMode1Nrg(); + using var ms = new MemoryStream(); + ms.Write(image); - NrgInPlaceModifier.WriteSector(stream, 20, payload); + var payload = new byte[Iso9660SectorSize]; + for (var i = 0; i < payload.Length; i++) payload[i] = (byte)(0xAA ^ (i & 0xFF)); - Assert.That(image.AsSpan(20 * Iso9660SectorSize, Iso9660SectorSize).ToArray(), Is.EqualTo(payload)); - Assert.That(image.AsSpan(trailer).ToArray(), Is.EqualTo(descriptorBytes)); + NrgInPlaceModifier.WriteSector(ms, lba: 25, payload); + + ms.Position = 25L * RawSectorSize + Mode1DataOffset; + var extract = new byte[Iso9660SectorSize]; + var got = 0; + while (got < extract.Length) { + var r = ms.Read(extract, got, extract.Length - got); + if (r == 0) break; + got += r; + } + Assert.That(extract, Is.EqualTo(payload)); } + // ── Boundary / contract ──────────────────────────────────────────── + [Test, Category("Boundary")] public void WriteSector_WrongUserDataLength_Throws() { - using var stream = new MemoryStream(BuildRawMode1Nrg(), writable: true); - Assert.Throws(() => NrgInPlaceModifier.WriteSector(stream, 20, new byte[100])); + using var ms = new MemoryStream(BuildRawMode1Nrg(), writable: true); + Assert.Throws(() => + NrgInPlaceModifier.WriteSector(ms, 20, new byte[100])); } [Test, Category("Boundary")] public void WriteSector_NegativeLba_Throws() { - using var stream = new MemoryStream(BuildRawMode1Nrg(), writable: true); + using var ms = new MemoryStream(BuildRawMode1Nrg(), writable: true); Assert.Throws(() => - NrgInPlaceModifier.WriteSector(stream, -1, new byte[Iso9660SectorSize])); + NrgInPlaceModifier.WriteSector(ms, -1, new byte[Iso9660SectorSize])); } [Test, Category("HappyPath")] public void Descriptor_AdvertisesCanModify() { - var descriptor = new NrgFormatDescriptor(); - Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + var d = new NrgFormatDescriptor(); + Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); } [Test, Category("HappyPath")] - public void Descriptor_ImplementsIArchiveModifiable() - => Assert.That(new NrgFormatDescriptor(), Is.InstanceOf()); + public void Descriptor_ImplementsIArchiveModifiable() { + Assert.That(new NrgFormatDescriptor(), Is.InstanceOf()); + } [Test, Category("HappyPath")] public void TryParseSectorEntryName_RoundTrips() { @@ -301,24 +460,37 @@ public void TryParseSectorEntryName_RoundTrips() { [Test, Category("Boundary")] public void TryParseSectorEntryName_BogusNames_Rejected() { - Assert.Multiple(() => { - Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("readme.txt", out _), Is.False); - Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("sector-abc.bin", out _), Is.False); - Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("sector-.bin", out _), Is.False); - Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("", out _), Is.False); - }); + Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("readme.txt", out _), Is.False); + Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("sector-abc.bin", out _), Is.False); + Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("sector-.bin", out _), Is.False); + Assert.That(NrgInPlaceModifier.TryParseSectorEntryName("", out _), Is.False); } [Test, Category("Boundary")] - public void DetectGeometry_StripsNer5Footer_FromDataAreaLength() { + public void Descriptor_Add_NonSectorEntry_IsRefusedAndPreservesImage() { + // This used to assert the image was untouched, and it was -- because the + // entry was dropped on the floor. Leaving the image alone is right; doing it + // without a word is not, and it made the shared "a fresh volume takes one + // more file" check pass on a volume that had taken nothing. var image = BuildRawMode1Nrg(); - using var stream = new MemoryStream(image, writable: true); - var geometry = NrgInPlaceModifier.DetectGeometry(stream); - - Assert.Multiple(() => { - Assert.That(geometry.SectorSize, Is.EqualTo(RawSectorSize)); - Assert.That(geometry.DataOffset, Is.EqualTo(Mode1DataOffset)); - Assert.That(geometry.DataAreaLength, Is.EqualTo(32 * RawSectorSize)); - }); + var original = (byte[])image.Clone(); + using var ms = new MemoryStream(image, writable: true); + + Assert.Throws(() => + ((IArchiveModifiable)new NrgFormatDescriptor()).Add(ms, [ + ArchiveInputInfo.InMemory("readme.txt", "hello"u8.ToArray()), + ])); + + Assert.That(image, Is.EqualTo(original), "a refused add must leave the image as it was"); + } + + [Test, Category("Boundary")] + public void DetectGeometry_StripsNer5Footer_FromDataAreaLength() { + var image = BuildRawMode1Nrg(sectorCount: 32); + using var ms = new MemoryStream(image, writable: true); + var geom = NrgInPlaceModifier.DetectGeometry(ms); + Assert.That(geom.SectorSize, Is.EqualTo(RawSectorSize)); + Assert.That(geom.DataOffset, Is.EqualTo(Mode1DataOffset)); + Assert.That(geom.DataAreaLength, Is.EqualTo(32 * RawSectorSize)); } } diff --git a/Compression.Tests/Nrg/NrgTests.cs b/Compression.Tests/Nrg/NrgTests.cs index 7d9e568a7..f15422dc1 100644 --- a/Compression.Tests/Nrg/NrgTests.cs +++ b/Compression.Tests/Nrg/NrgTests.cs @@ -1,290 +1,125 @@ -using System.Buffers.Binary; -using Compression.Registry; -using FileFormat.Nrg; - namespace Compression.Tests.Nrg; [TestFixture] public class NrgTests { - private static byte[] CreateNrg(params (string Name, byte[] Data)[] files) { - var descriptor = new NrgFormatDescriptor(); - using var image = new MemoryStream(); - ((IArchiveCreatable)descriptor).Create( - image, - files.Select(static file => ArchiveInputInfo.InMemory(file.Name, file.Data)).ToArray(), - new FormatCreateOptions()); - return image.ToArray(); - } - - private static byte[] ExtractCanonicalIso(string fileName, byte[] data) { - var canonical = CreateNrg((fileName, data)); - var trailerOffset = checked((int)BinaryPrimitives.ReadUInt64BigEndian(canonical.AsSpan(canonical.Length - 8))); - return canonical.AsSpan(0, trailerOffset).ToArray(); - } - - private static byte[] BuildOffsetTrackNrg(string fileName, byte[] data, int prefixLength = 4096) { - var iso = ExtractCanonicalIso(fileName, data); - - using var output = new MemoryStream(); - output.Write(new byte[prefixLength]); - output.Write(iso); - var trailerOffset = checked((ulong)output.Position); - - Span etn2 = stackalloc byte[32]; - BinaryPrimitives.WriteUInt64BigEndian(etn2, checked((ulong)prefixLength)); - BinaryPrimitives.WriteUInt64BigEndian(etn2[8..], checked((ulong)iso.LongLength)); - etn2[19] = 0x00; - WriteChunk(output, "ETN2"u8, etn2); - - WriteCommonTrailerChunks(output); - Span footer = stackalloc byte[12]; - "NER5"u8.CopyTo(footer); - BinaryPrimitives.WriteUInt64BigEndian(footer[4..], trailerOffset); - output.Write(footer); - return output.ToArray(); - } - - private static byte[] BuildLegacyOffsetTrackNrg(string fileName, byte[] data, int prefixLength = 4096) { - var iso = ExtractCanonicalIso(fileName, data); - - using var output = new MemoryStream(); - output.Write(new byte[prefixLength]); - output.Write(iso); - var trailerOffset = checked((uint)output.Position); - - Span etnf = stackalloc byte[20]; - BinaryPrimitives.WriteUInt32BigEndian(etnf, checked((uint)prefixLength)); - BinaryPrimitives.WriteUInt32BigEndian(etnf[4..], checked((uint)iso.Length)); - etnf[11] = 0x00; - WriteChunk(output, "ETNF"u8, etnf); - - WriteCommonTrailerChunks(output); - - Span footer = stackalloc byte[8]; - "NERO"u8.CopyTo(footer); - BinaryPrimitives.WriteUInt32BigEndian(footer[4..], trailerOffset); - output.Write(footer); - return output.ToArray(); - } - - private static void WriteCommonTrailerChunks(Stream output) { - Span sinf = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32BigEndian(sinf, 1); - WriteChunk(output, "SINF"u8, sinf); - - Span mtyp = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32BigEndian(mtyp, 0x00000400); - WriteChunk(output, "MTYP"u8, mtyp); - WriteChunk(output, "END!"u8, ReadOnlySpan.Empty); - } - - private static void WriteChunk(Stream output, ReadOnlySpan id, ReadOnlySpan payload) { - Span header = stackalloc byte[8]; - id.CopyTo(header); - BinaryPrimitives.WriteUInt32BigEndian(header[4..], checked((uint)payload.Length)); - output.Write(header); - output.Write(payload); - } - - private static byte[] ReadFile(byte[] image, string leaf) { - using var stream = new MemoryStream(image, writable: false); - using var reader = new NrgReader(stream); - var entry = reader.Entries.Single(e => !e.IsDirectory && e.Name.Equals(leaf, StringComparison.OrdinalIgnoreCase)); - return reader.Extract(entry); + // Builds a minimal NRG v2 image: flat ISO sectors followed by the NER5 footer. + // The footer is 12 bytes: "NER5" + uint64 BE offset to chunk table (appended after data). + private static byte[] BuildNrg(string fileName, byte[] fileData) { + const int sectorSize = 2048; + const int pvdLba = 16; + const int rootDirLba = 18; + const int fileLba = 19; + + var totalSectors = fileLba + 2; + var isoBuf = new byte[totalSectors * sectorSize]; + + // File data + fileData.AsSpan().CopyTo(isoBuf.AsSpan(fileLba * sectorSize)); + + // Root directory + var dirPos = rootDirLba * sectorSize; + var dot = new byte[34]; dot[0] = 34; dot[2] = rootDirLba; dot[25] = 0x02; dot[32] = 1; dot[33] = 0x00; + dot.AsSpan().CopyTo(isoBuf.AsSpan(dirPos)); dirPos += 34; + var dotdot = new byte[34]; dotdot[0] = 34; dotdot[2] = rootDirLba; dotdot[25] = 0x02; dotdot[32] = 1; dotdot[33] = 0x01; + dotdot.AsSpan().CopyTo(isoBuf.AsSpan(dirPos)); dirPos += 34; + + var isoName = fileName.ToUpperInvariant() + ";1"; + var idLen = (byte)isoName.Length; + var recLen = (byte)(33 + idLen + ((33 + idLen) % 2)); + var rec = new byte[recLen]; + rec[0] = recLen; rec[2] = (byte)fileLba; rec[6] = (byte)fileLba; + var sz = (uint)fileData.Length; BitConverter.GetBytes(sz).CopyTo(rec, 10); // LE at offset 10 + rec[32] = idLen; System.Text.Encoding.ASCII.GetBytes(isoName).CopyTo(rec, 33); + rec.AsSpan().CopyTo(isoBuf.AsSpan(dirPos)); + dirPos += recLen; + + var pvd = new byte[sectorSize]; + pvd[0] = 1; System.Text.Encoding.ASCII.GetBytes("CD001").CopyTo(pvd, 1); pvd[6] = 1; + pvd[156] = 34; pvd[158] = rootDirLba; pvd[162] = rootDirLba; + pvd[166] = (byte)(dirPos - rootDirLba * sectorSize); pvd[170] = pvd[166]; + pvd[156 + 25] = 0x02; pvd[156 + 32] = 1; + pvd.AsSpan().CopyTo(isoBuf.AsSpan(pvdLba * sectorSize)); + + // NRG v2 footer: "NER5" + uint64 BE (chunk table offset = end of data area) + var chunkOffset = (ulong)isoBuf.Length; + var footer = new byte[12]; + footer[0] = (byte)'N'; footer[1] = (byte)'E'; footer[2] = (byte)'R'; footer[3] = (byte)'5'; + footer[4] = (byte)(chunkOffset >> 56); + footer[5] = (byte)(chunkOffset >> 48); + footer[6] = (byte)(chunkOffset >> 40); + footer[7] = (byte)(chunkOffset >> 32); + footer[8] = (byte)(chunkOffset >> 24); + footer[9] = (byte)(chunkOffset >> 16); + footer[10] = (byte)(chunkOffset >> 8); + footer[11] = (byte)chunkOffset; + + var result = new byte[isoBuf.Length + footer.Length]; + isoBuf.AsSpan().CopyTo(result); + footer.AsSpan().CopyTo(result.AsSpan(isoBuf.Length)); + return result; } [Test, Category("HappyPath")] public void Read_Nrg_DetectsVersion2Footer() { - using var stream = new MemoryStream(CreateNrg(("README.TXT", "NRG content"u8.ToArray()))); - using var reader = new NrgReader(stream); - Assert.That(reader.Version, Is.EqualTo(2)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Create_WritesRealV2DaoChunkChain() { - var image = CreateNrg(("DATA.BIN", "nero-payload"u8.ToArray())); - Assert.That(image.AsSpan(image.Length - 12, 4).ToArray(), Is.EqualTo("NER5"u8.ToArray())); - - var trailerOffset = checked((int)BinaryPrimitives.ReadUInt64BigEndian(image.AsSpan(image.Length - 8))); - Assert.That(trailerOffset, Is.LessThan(image.Length - 12)); - Assert.That(trailerOffset % 2048, Is.Zero); - - var position = trailerOffset; - var cuex = ReadChunk(image, ref position, "CUEX", 32).ToArray(); - Assert.Multiple(() => { - Assert.That(cuex[0], Is.EqualTo(0x41)); - Assert.That(cuex[1], Is.EqualTo(0x00)); - Assert.That(cuex[2], Is.EqualTo(0x00)); - Assert.That(unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(cuex.AsSpan(4))), Is.EqualTo(-150)); - Assert.That(cuex[9], Is.EqualTo(0x01)); - Assert.That(cuex[10], Is.EqualTo(0x00)); - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex.AsSpan(12)), Is.Zero); - Assert.That(cuex[17], Is.EqualTo(0x01)); - Assert.That(cuex[18], Is.EqualTo(0x01)); - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex.AsSpan(20)), Is.Zero); - Assert.That(cuex[25], Is.EqualTo(0xAA)); - Assert.That(cuex[26], Is.EqualTo(0x01)); - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(cuex.AsSpan(28)), Is.EqualTo((uint)(trailerOffset / 2048))); - }); - - var daox = ReadChunk(image, ref position, "DAOX", 64).ToArray(); - var track = daox.AsSpan(22).ToArray(); - Assert.Multiple(() => { - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(daox), Is.EqualTo(64)); - Assert.That(daox[20], Is.EqualTo(1)); - Assert.That(daox[21], Is.EqualTo(1)); - Assert.That(BinaryPrimitives.ReadUInt16BigEndian(track.AsSpan(12)), Is.EqualTo(2048)); - Assert.That(track[14], Is.Zero, "cooked Mode 1"); - Assert.That(BinaryPrimitives.ReadUInt16BigEndian(track.AsSpan(16)), Is.EqualTo(1)); - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track.AsSpan(18)), Is.Zero); - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track.AsSpan(26)), Is.Zero); - Assert.That(BinaryPrimitives.ReadUInt64BigEndian(track.AsSpan(34)), Is.EqualTo((ulong)trailerOffset)); - }); - - var sinf = ReadChunk(image, ref position, "SINF", 4).ToArray(); - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(sinf), Is.EqualTo(1)); - - var mtyp = ReadChunk(image, ref position, "MTYP", 4).ToArray(); - Assert.That(BinaryPrimitives.ReadUInt32BigEndian(mtyp), Is.EqualTo(0x00000400)); - - _ = ReadChunk(image, ref position, "END!", 0); - Assert.That(position, Is.EqualTo(image.Length - 12)); - } + var data = "NRG content"u8.ToArray(); + var nrg = BuildNrg("readme.txt", data); + using var ms = new MemoryStream(nrg); - private static ReadOnlySpan ReadChunk(byte[] image, ref int position, string id, int expectedLength) { - Assert.That(System.Text.Encoding.ASCII.GetString(image, position, 4), Is.EqualTo(id)); - var length = checked((int)BinaryPrimitives.ReadUInt32BigEndian(image.AsSpan(position + 4, 4))); - Assert.That(length, Is.EqualTo(expectedLength)); - var payload = image.AsSpan(position + 8, length); - position += 8 + length; - return payload; + var r = new FileFormat.Nrg.NrgReader(ms); + Assert.That(r.Version, Is.EqualTo(2)); } - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Read_UsesEtn2TrackOffset_InsteadOfAssumingOffsetZero() { - var payload = "track-offset"u8.ToArray(); - var image = BuildOffsetTrackNrg("OFFSET.BIN", payload); - Assert.That(ReadFile(image, "OFFSET.BIN"), Is.EqualTo(payload)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Read_LegacyNeroEtnf_Uses32BitTrackOffset() { - var payload = "legacy-track-offset"u8.ToArray(); - var image = BuildLegacyOffsetTrackNrg("LEGACY.BIN", payload); - - using var stream = new MemoryStream(image, writable: false); - using var reader = new NrgReader(stream); - Assert.That(reader.Version, Is.EqualTo(1)); - var entry = reader.Entries.Single(e => !e.IsDirectory && e.Name.Equals("LEGACY.BIN", StringComparison.OrdinalIgnoreCase)); - Assert.That(reader.Extract(entry), Is.EqualTo(payload)); - } - - [Test, Category("Regression")] - public void Read_TruncatedChunkLength_FailsClosedAndFallsBackToLegacyTrackProbe() { - var canonical = CreateNrg(("SAFE.BIN", "safe"u8.ToArray())); - var trailer = checked((int)BinaryPrimitives.ReadUInt64BigEndian(canonical.AsSpan(canonical.Length - 8))); - var iso = canonical.AsSpan(0, trailer).ToArray(); - - using var output = new MemoryStream(); - output.Write(iso); - var malformedTrailer = checked((ulong)output.Position); - output.Write("ETN2"u8); - Span absurdLength = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32BigEndian(absurdLength, uint.MaxValue); - output.Write(absurdLength); - Span footer = stackalloc byte[12]; - "NER5"u8.CopyTo(footer); - BinaryPrimitives.WriteUInt64BigEndian(footer[4..], malformedTrailer); - output.Write(footer); - - Assert.That(ReadFile(output.ToArray(), "SAFE.BIN"), Is.EqualTo("safe"u8.ToArray())); - } - - [Test, Category("RoundTrip")] - public void Create_PreservesDirectoryPaths() { - var image = CreateNrg(("DIR/SUB/FILE.BIN", "nested"u8.ToArray())); - using var stream = new MemoryStream(image); - using var reader = new NrgReader(stream); - - Assert.That(reader.Entries.Any(e => !e.IsDirectory && e.FullPath.Equals("DIR/SUB/FILE.BIN", StringComparison.OrdinalIgnoreCase)), Is.True); - Assert.That(ReadFile(image, "FILE.BIN"), Is.EqualTo("nested"u8.ToArray())); - } - - [Test, Category("RoundTrip")] - public void Modify_AddReplaceRemove_RoundTripsNamedIsoEntries() { - var descriptor = new NrgFormatDescriptor(); - using var image = new MemoryStream(); - image.Write(CreateNrg(("SEED.BIN", "seed"u8.ToArray()))); - image.Position = 0; - - ((IArchiveModifiable)descriptor).Add(image, [ArchiveInputInfo.InMemory("PROBE.BIN", "probe"u8.ToArray())]); - Assert.That(ReadFile(image.ToArray(), "SEED.BIN"), Is.EqualTo("seed"u8.ToArray())); - Assert.That(ReadFile(image.ToArray(), "PROBE.BIN"), Is.EqualTo("probe"u8.ToArray())); - - ((IArchiveModifiable)descriptor).Add(image, [ArchiveInputInfo.InMemory("PROBE.BIN", "replaced"u8.ToArray())]); - Assert.That(ReadFile(image.ToArray(), "PROBE.BIN"), Is.EqualTo("replaced"u8.ToArray())); - - ((IArchiveModifiable)descriptor).Remove(image, ["PROBE.BIN"]); - using var after = new MemoryStream(image.ToArray()); - using var reader = new NrgReader(after); - Assert.That(reader.Entries.Any(e => !e.IsDirectory && e.Name.Equals("PROBE.BIN", StringComparison.OrdinalIgnoreCase)), Is.False); - Assert.That(ReadFile(image.ToArray(), "SEED.BIN"), Is.EqualTo("seed"u8.ToArray())); - } - - [Test, Category("RoundTrip")] - public void Maintenance_RebuildsOffsetImage_WithoutChangingPayload() { - var payload = Enumerable.Range(0, 1000).Select(static i => (byte)(i * 31)).ToArray(); - var source = BuildOffsetTrackNrg("KEEP.BIN", payload, prefixLength: 8192); - var descriptor = new NrgFormatDescriptor(); - - using var defrag = new MemoryStream(); - defrag.Write(source); - defrag.Position = 0; - ((IArchiveDefragmentable)descriptor).Defragment(defrag); - Assert.That(ReadFile(defrag.ToArray(), "KEEP.BIN"), Is.EqualTo(payload)); + [Test, Category("HappyPath")] + public void Read_Nrg_ListsFile() { + var data = "Nero disc data"u8.ToArray(); + var nrg = BuildNrg("file.dat", data); + using var ms = new MemoryStream(nrg); - using var input = new MemoryStream(source); - using var shrunk = new MemoryStream(); - ((IArchiveShrinkable)descriptor).Shrink(input, shrunk); - Assert.That(shrunk.Length, Is.LessThan(source.Length)); - Assert.That(ReadFile(shrunk.ToArray(), "KEEP.BIN"), Is.EqualTo(payload)); + var r = new FileFormat.Nrg.NrgReader(ms); + Assert.That(r.Entries, Has.Count.GreaterThan(0)); + var file = r.Entries.FirstOrDefault(e => !e.IsDirectory); + Assert.That(file, Is.Not.Null); + Assert.That(file!.Size, Is.EqualTo(data.Length)); } - [Test, Category("RoundTrip")] - public void Purge_LeavesValidEmptyNrg() { - var descriptor = new NrgFormatDescriptor(); - using var image = new MemoryStream(); - image.Write(CreateNrg(("DELETE.ME", "gone"u8.ToArray()))); - image.Position = 0; + [Test, Category("HappyPath")] + public void Read_Nrg_ExtractReturnsData() { + var data = new byte[128]; + Random.Shared.NextBytes(data); + var nrg = BuildNrg("sample.bin", data); + using var ms = new MemoryStream(nrg); - ((IArchivePurgeable)descriptor).Purge(image); + var r = new FileFormat.Nrg.NrgReader(ms); + var file = r.Entries.FirstOrDefault(e => !e.IsDirectory); + Assert.That(file, Is.Not.Null); - image.Position = 0; - using var reader = new NrgReader(image, leaveOpen: true); - Assert.That(reader.Version, Is.EqualTo(2)); - Assert.That(reader.Entries.Where(static e => !e.IsDirectory), Is.Empty); + var extracted = r.Extract(file!); + Assert.That(extracted[..data.Length], Is.EqualTo(data)); } [Test, Category("HappyPath")] - public void Descriptor_AdvertisesOnlySupportedMaintenanceVerbs() { - var descriptor = new NrgFormatDescriptor(); - Assert.Multiple(() => { - Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); - Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); - Assert.That(descriptor, Is.InstanceOf()); - Assert.That(descriptor, Is.InstanceOf()); - Assert.That(descriptor, Is.InstanceOf()); - Assert.That(descriptor, Is.Not.InstanceOf()); - Assert.That(descriptor, Is.Not.InstanceOf()); - }); + public void Read_NoFooter_VersionIsZero() { + // Stream with no NRG footer — version should be 0 + using var ms = new MemoryStream(new byte[2352 * 32]); + var r = new FileFormat.Nrg.NrgReader(ms); + Assert.That(r.Version, Is.EqualTo(0)); } [Test, Category("HappyPath")] - public void Read_NoFooter_VersionIsZero() { - using var stream = new MemoryStream(new byte[2352 * 32]); - using var reader = new NrgReader(stream); - Assert.That(reader.Version, Is.Zero); + public void NrgEntry_Properties() { + var entry = new FileFormat.Nrg.NrgEntry { + Name = "AUTORUN.INF", + FullPath = "AUTORUN.INF", + IsDirectory = false, + Size = 128, + StartLba = 20, + }; + Assert.That(entry.Name, Is.EqualTo("AUTORUN.INF")); + Assert.That(entry.FullPath, Is.EqualTo("AUTORUN.INF")); + Assert.That(entry.IsDirectory, Is.False); + Assert.That(entry.Size, Is.EqualTo(128)); + Assert.That(entry.StartLba, Is.EqualTo(20)); } [Test, Category("HappyPath")] @@ -292,4 +127,38 @@ public void Detect_ByExtension() { var format = Compression.Lib.FormatDetector.DetectByExtension("disc.nrg"); Assert.That(format, Is.EqualTo(Compression.Lib.FormatDetector.Format.Nrg)); } + + [Test, Category("HappyPath")] + public void Descriptor_ReportsWormCapability() { + var d = new FileFormat.Nrg.NrgFormatDescriptor(); + Assert.That(d.Capabilities.HasFlag(Compression.Registry.FormatCapabilities.CanCreate), Is.True); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Create_HasNer5Footer_AndRoundTrips() { + var payload = "nero-payload"u8.ToArray(); + var tmp = Path.GetTempFileName(); + try { + File.WriteAllBytes(tmp, payload); + var d = new FileFormat.Nrg.NrgFormatDescriptor(); + using var ms = new MemoryStream(); + ((Compression.Registry.IArchiveCreatable)d).Create( + ms, + [new Compression.Registry.ArchiveInputInfo(tmp, "data.bin", false)], + new Compression.Registry.FormatCreateOptions()); + + // Verify the NER5 footer was emitted (last 12 bytes: "NER5" + uint64 BE). + var bytes = ms.ToArray(); + Assert.That(bytes[^12..^8], Is.EqualTo(new byte[] { (byte)'N', (byte)'E', (byte)'R', (byte)'5' })); + + ms.Position = 0; + var r = new FileFormat.Nrg.NrgReader(ms); + Assert.That(r.Version, Is.EqualTo(2)); + var fileEntry = r.Entries.FirstOrDefault(e => !e.IsDirectory && e.Name.StartsWith("DATA")); + Assert.That(fileEntry, Is.Not.Null); + Assert.That(r.Extract(fileEntry!)[..payload.Length], Is.EqualTo(payload)); + } finally { + File.Delete(tmp); + } + } } diff --git a/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs b/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs index 7d8f55b70..25da12c5e 100644 --- a/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs +++ b/FileFormats/FileFormat.Nrg/NrgFormatDescriptor.cs @@ -5,208 +5,144 @@ namespace FileFormat.Nrg; /// -/// Nero Burning ROM NRG disc image — a sector stream followed by a chunked -/// session/track descriptor and a trailing NERO/NER5 footer. +/// Nero Burning ROM NRG disc image — trailing NER5/NERO footer pointing at a chunked session/track descriptor area. /// /// References: /// -/// https://cdemu.sourceforge.io — CDEmu / libMirage NRG parser, used as a behavioural oracle for the reverse-engineered chunk layout -/// https://problemkaputt.de/psx-spx.htm — independently documented NRG CUEX/DAOX/ETN structures -/// No official public Nero specification is available; the format is proprietary and reverse-engineered +/// https://cdemu.sourceforge.io — CDEmu / libMirage — its NRG parser is the de-facto format documentation +/// https://en.wikipedia.org/wiki/NRG_(file_format) — Wikipedia +/// No official specification — proprietary Nero format, reverse-engineered /// /// -public sealed class NrgFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, - IArchiveModifiable, IArchiveDefragmentable, IArchiveShrinkable { +public sealed class NrgFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable { - /// Gets the id. - public string Id => "Nrg"; - /// Gets the display name. + /// + /// Gets the id. + /// + public string Id => "Nrg"; + /// + /// Gets the display name. + /// public string DisplayName => "NRG"; - - /// Gets the category. + /// + /// Gets the category. + /// public FormatCategory Category => FormatCategory.Archive; - - /// Gets the capabilities. + /// + /// Gets the capabilities. + /// public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; - - /// Gets the default extension. + /// + /// Gets the default extension. + /// public string DefaultExtension => ".nrg"; - - /// Gets the extensions. + /// + /// Gets the extensions. + /// public IReadOnlyList Extensions => [".nrg"]; - - /// Gets the compound extensions. + /// + /// Gets the compound extensions. + /// public IReadOnlyList CompoundExtensions => []; - // NRG magic is a footer signature ("NER5" or "NERO" at a variable offset from EOF), // which cannot be represented as a fixed-offset MagicSignature. - /// Gets the magic signatures. + // Detection relies on the file extension and footer heuristic in NrgReader. + /// + /// Gets the magic signatures. + /// public IReadOnlyList MagicSignatures => []; - - /// Gets the methods. - public IReadOnlyList Methods => [ - new("iso9660", "ISO 9660"), - new("cdda", "CD-DA / mixed-mode tracks"), - ]; - - /// Gets the tar compression format id. + /// + /// Gets the methods. + /// + public IReadOnlyList Methods => [new("iso9660", "ISO 9660")]; + /// + /// Gets the tar compression format id. + /// public string? TarCompressionFormatId => null; - - /// Gets the family. + /// + /// Gets the family. + /// public AlgorithmFamily Family => AlgorithmFamily.Archive; + /// + /// Gets the description. + /// + public string Description => "Nero Burning ROM disc image (R/W via in-place sector rewrite at fixed offsets; inner ISO 9660 directory mutation delegated to FileSystem.Iso; multi-track DAOI/CUEX layouts deferred)"; - /// Gets the description. - public string Description => - "Nero Burning ROM disc image (NRG v1/v2 reader; v2 DAO multi-session/multi-track/audio writer; named ISO edits use verified rebuild)"; - - /// Lists the ISO 9660 entries from the first readable data track in the supplied container. + /// + /// Lists the entries in the supplied container. + /// public List List(Stream stream, string? password) { - using var reader = new NrgReader(stream, leaveOpen: true); - return reader.Entries.Select((entry, index) => new ArchiveEntryInfo(index, entry.FullPath, entry.Size, - entry.Size, "iso9660", entry.IsDirectory, false, null)).ToList(); + var r = new NrgReader(stream, leaveOpen: true); + return r.Entries.Select((e, i) => new ArchiveEntryInfo(i, e.FullPath, e.Size, + e.Size, "iso9660", e.IsDirectory, false, null)).ToList(); } - /// Extracts ISO 9660 entries from the first readable data track. + /// + /// Decodes the supplied input. + /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - using var reader = new NrgReader(stream, leaveOpen: true); - foreach (var entry in reader.Entries) { - if (entry.IsDirectory) - continue; - if (files != null && !MatchesFilter(entry.FullPath, files)) - continue; - WriteFile(outputDir, entry.FullPath, reader.Extract(entry)); + var r = new NrgReader(stream, leaveOpen: true); + foreach (var e in r.Entries) { + if (e.IsDirectory) continue; + if (files != null && !MatchesFilter(e.FullPath, files)) continue; + WriteFile(outputDir, e.FullPath, r.Extract(e)); } } /// - /// Creates the generic archive API profile: one DAO session containing one cooked - /// Mode-1 ISO 9660 track. Call - /// directly for multi-session, mixed data/audio, pregap, MCN, ISRC, CD-TEXT or raw-sector authoring. + /// Performs the create operation. /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { - ArgumentNullException.ThrowIfNull(output); - ArgumentNullException.ThrowIfNull(inputs); - + // WORM: ISO 9660 image followed by an NRG v2 ("NER5") footer. The reader + // only uses the footer to determine version; the chunk-table offset isn't + // dereferenced for ISO extraction, so we point it at end-of-file. var iso = new FileSystem.Iso.IsoWriter(); - foreach (var input in inputs.Where(static input => !input.IsDirectory)) - iso.AddFile(input.ArchiveName.Replace('\\', '/'), input.ReadContent()); - - NrgWriter.Write(output, new NrgDiscDefinition([ - new NrgSessionDefinition([ - new NrgTrackDefinition(NrgTrackMode.Mode1, iso.Build()), - ]), - ])); + foreach (var (name, data) in FlatFiles(inputs)) + iso.AddFile(name, data); + var image = iso.Build(); + output.Write(image); + // Footer: "NER5" (4 bytes) + uint64 BE chunk-table offset. + Span footer = stackalloc byte[12]; + footer[0] = (byte)'N'; footer[1] = (byte)'E'; footer[2] = (byte)'R'; footer[3] = (byte)'5'; + System.Buffers.Binary.BinaryPrimitives.WriteUInt64BigEndian(footer[4..], (ulong)image.Length); + output.Write(footer); } + // ── IArchiveModifiable ────────────────────────────────────────────── + /// - /// Adds/replaces named ISO entries through a verified rebuild. Multi-track or audio NRGs are - /// deliberately refused here because flattening them to the generic single-ISO create profile - /// would destroy disc structure. + /// Rewrites raw CD sectors in place. Inputs whose ArchiveName matches + /// sector-NNNNNN.bin are written at the fixed byte offset + /// lba * sectorSize + dataOffset; everything outside the touched + /// 2 048-byte user-data region — including the trailing NRG footer — stays + /// byte-identical (the footer migrates with the new EOF when the data area + /// grows past the previous end). + /// + /// Inputs not matching the synthetic sector schema are skipped — + /// inner-ISO 9660 directory mutation is delegated to FileSystem.Iso + /// and is out of scope for the sector-rewrite modifier. /// public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(inputs); - EnsureNamedFileMutationProfile(archive); - RebuildVerb.EditViaRebuild(archive, this, this, tmpDir => { - foreach (var input in inputs) { - if (input.IsDirectory || string.IsNullOrEmpty(input.ArchiveName)) - continue; - var destination = Path.Combine(tmpDir, input.ArchiveName.Replace('/', Path.DirectorySeparatorChar)); - var parent = Path.GetDirectoryName(destination); - if (!string.IsNullOrEmpty(parent)) - Directory.CreateDirectory(parent); - File.WriteAllBytes(destination, input.ReadContent()); - } - }); - } - - /// Removes named ISO entries through the same profile-gated verified rebuild. - public void Remove(Stream archive, string[] entryNames) { - ArgumentNullException.ThrowIfNull(entryNames); - EnsureNamedFileMutationProfile(archive); - var skip = new HashSet(entryNames, StringComparer.OrdinalIgnoreCase); - RebuildVerb.EditViaRebuild(archive, this, this, tmpDir => { - foreach (var file in Directory.GetFiles(tmpDir, "*", SearchOption.AllDirectories)) { - var relative = Path.GetRelativePath(tmpDir, file).Replace('\\', '/'); - if (skip.Contains(relative) || skip.Contains(Path.GetFileName(relative))) - File.Delete(file); - } - }); - } - - /// Purges the single-data-track R/W profile to a valid empty NRG. - public void Purge(Stream archive) { - EnsureNamedFileMutationProfile(archive); - RebuildVerb.PurgeViaModifier(archive, this, this); - } - - /// Rebuild-defragments the single-data-track R/W profile. - public void Defragment(Stream archive) { - EnsureNamedFileMutationProfile(archive); - RebuildVerb.RebuildInPlace(archive, this, this); - } - - /// Progress-reporting rebuild defrag for the single-data-track R/W profile. - public void Defragment(Stream archive, DefragOptions options) { - ArgumentNullException.ThrowIfNull(options); - if (options.Mode != DefragMode.ConsolidateAtStart) - throw new NotSupportedException($"NRG rebuild defrag supports only {DefragMode.ConsolidateAtStart}."); - EnsureNamedFileMutationProfile(archive); - RebuildVerb.RebuildInPlace(archive, this, this, - onProgress: options.OnProgress, - cancellationToken: options.CancellationToken); + NrgInPlaceModifier.AddOrReplaceSectors(archive, + inputs.Where(i => !i.IsDirectory).Select(i => (i.ArchiveName, i.ReadContent()))); } /// - /// Tight-packs the single-data-track profile. Multi-track/audio images are copied through - /// unchanged rather than being flattened into one ISO track. + /// Zeros the 2 048-byte user-data region of each named sector. Sector + /// framing bytes (sync / address / mode / EDC) on raw geometries and the + /// trailing NRG footer are preserved so the LBA-to-offset map and the rest + /// of the image remain byte-identical. /// - public void Shrink(Stream input, Stream output) { - ArgumentNullException.ThrowIfNull(input); - ArgumentNullException.ThrowIfNull(output); - var profile = NrgStructureInspector.Inspect(input); - if (!profile.IsSingleDataTrack) { - CopyThrough(input, output); - return; - } - - using var rebuilt = RebuildVerb.CreateScratchStream(); - var useRebuilt = false; - try { - RebuildVerb.RebuildToStream(input, rebuilt, this, this); - useRebuilt = rebuilt.Length > 0 && rebuilt.Length < input.Length; - } catch { - useRebuilt = false; - } - - output.Position = 0; - output.SetLength(0); - if (useRebuilt) { - rebuilt.Position = 0; - rebuilt.CopyTo(output); - } else { - input.Position = 0; - input.CopyTo(output); - } - } - - private static void CopyThrough(Stream input, Stream output) { - input.Position = 0; - output.Position = 0; - output.SetLength(0); - input.CopyTo(output); - } - - private static void EnsureNamedFileMutationProfile(Stream archive) { + public void Remove(Stream archive, string[] entryNames) { ArgumentNullException.ThrowIfNull(archive); - var profile = NrgStructureInspector.Inspect(archive); - if (!profile.IsSingleDataTrack) - throw new NotSupportedException( - "Named-file mutation/defrag/purge is supported only for a single data-track NRG. " + - "Multi-session, mixed-mode and audio images are authorable but are not flattened during file-level maintenance."); + ArgumentNullException.ThrowIfNull(entryNames); + NrgInPlaceModifier.RemoveSectors(archive, entryNames); } } diff --git a/FileFormats/FileFormat.Nrg/NrgInPlaceModifier.cs b/FileFormats/FileFormat.Nrg/NrgInPlaceModifier.cs index 948c549b0..f3f53ec7b 100644 --- a/FileFormats/FileFormat.Nrg/NrgInPlaceModifier.cs +++ b/FileFormats/FileFormat.Nrg/NrgInPlaceModifier.cs @@ -1,247 +1,333 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; using System.Globalization; namespace FileFormat.Nrg; /// -/// Low-level fixed-LBA sector editor for single-track NRG images whose ISO data -/// track begins at file offset zero. This is intentionally narrower than the -/// descriptor's public file-level R/W path: ordinary file CRUD uses a verified -/// extract/edit/re-create rebuild so ISO directory metadata stays coherent. +/// In-place sector-rewrite modifier for a Nero Burning ROM NRG disc image. +/// Operates at the raw 2 048-byte user-data region of each CD sector at the +/// fixed byte offset lba * sectorSize + dataOffset, where +/// sectorSize and dataOffset are the geometry detected from the +/// data area (raw 2 352 Mode 1, raw 2 352 Mode 2 Form 1, 2 336-byte sectors, +/// or flat 2 048-byte cooked sectors). +/// +/// NRG framing. An NRG image is a stream of CD sectors followed +/// by a footer at EOF identifying the format version: +/// +/// NRG v2: last 12 bytes — "NER5" + uint64 BE chunk-table offset. +/// NRG v1: last 8 bytes — "NERO" + uint32 BE chunk-table offset. +/// +/// The footer is preserved byte-identical across in-place rewrites and is +/// relocated past the new EOF whenever the data area grows. +/// +/// Scope. Rewrites only the user-data bytes inside an existing +/// sector or appends a brand-new sector at the end of the data area. It does +/// not understand the inner ISO 9660 directory structure — that is the +/// job of / its reader. Synthetic entry +/// names of the form sector-NNNNNN.bin address a single sector LBA. +/// Multi-track DAOI/CUEX layouts are not parsed — the modifier treats the +/// stream as a single track of sectors at flat LBA offsets. Sync pattern +/// (12 B), 3-byte address, 1-byte mode, and the EDC/ECC tail of raw sectors +/// are preserved on rewrite and synthesised on append. +/// +/// True in-place. Writes touch only the 2 048-byte user-data +/// region of the targeted sector. Bytes outside that region — header bytes of +/// the same sector, every untouched sector, the system area (LBA 0-15), the +/// PVD at LBA 16, the ISO root directory, and the trailing NRG footer — stay +/// byte-identical at their original byte offsets (the footer migrates to +/// follow the new EOF when the data area grows). /// public static class NrgInPlaceModifier { + + /// How this format names itself when it has to refuse something. private const string Label = "NRG"; + + private const int Iso9660SectorSize = 2048; private const int RawSectorSize = 2352; private const int SectorSize2336 = 2336; private const int PvdLba = 16; private const int Mode1DataOffset = 16; private const int Mode2Form1DataOffset = 24; + private const int Ner5FooterSize = 12; private const int NeroFooterSize = 8; - private static ReadOnlySpan Sync => [ + private static readonly byte[] MagicNer5 = [(byte)'N', (byte)'E', (byte)'R', (byte)'5']; + private static readonly byte[] MagicNero = [(byte)'N', (byte)'E', (byte)'R', (byte)'O']; + + /// The 12-byte CD-ROM sync pattern (00 FF*10 00) prefixed to every + /// raw 2 352-byte sector. + private static readonly byte[] Sync = [ 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, ]; /// - /// Detected sector geometry. is the byte offset of - /// the 2,048-byte user-data region. is the byte - /// offset at which the NRG descriptor begins, not merely EOF minus the footer. + /// Detected on-disk sector geometry for an NRG image. + /// is the byte offset within a sector where the 2 048 B of ISO user data + /// begins. excludes the trailing NRG footer + /// (12 bytes for v2, 8 bytes for v1) when present. /// public readonly record struct SectorGeometry(int SectorSize, int DataOffset, long DataAreaLength); - private readonly record struct Trailer(long Offset, bool HasNrgFooter); - /// - /// Detects the single-track geometry from the ISO PVD at LBA 16. The NERO/ - /// NER5 footer's trailer pointer is honoured, so chunk metadata is never - /// mistaken for sector data. + /// Detects the sector geometry of the same way + /// does — by probing for the CD001 PVD + /// signature at LBA 16 inside the data area. Falls back to raw Mode 1 + /// (2 352 / 16) when no probe succeeds. NRG v2 ("NER5") and v1 ("NERO") + /// footers are excluded from the data area. /// public static SectorGeometry DetectGeometry(Stream image) { ArgumentNullException.ThrowIfNull(image); - if (!image.CanRead || !image.CanSeek) - throw new ArgumentException("NRG sector editing requires a readable, seekable stream.", nameof(image)); - - var trailer = ReadTrailer(image); - var dataLength = trailer.Offset; - if (TryProbe(image, RawSectorSize, Mode1DataOffset, dataLength)) - return new(RawSectorSize, Mode1DataOffset, dataLength); - if (TryProbe(image, RawSectorSize, Mode2Form1DataOffset, dataLength)) - return new(RawSectorSize, Mode2Form1DataOffset, dataLength); - if (TryProbe(image, SectorSize2336, 8, dataLength)) - return new(SectorSize2336, 8, dataLength); - if (TryProbe(image, Iso9660SectorSize, 0, dataLength)) - return new(Iso9660SectorSize, 0, dataLength); - - throw new NotSupportedException( - $"{Label}: the fixed-LBA editor supports only a single ISO data track beginning at file offset zero."); + var dataLen = DetectDataAreaLength(image); + if (TryProbe(image, RawSectorSize, Mode1DataOffset, dataLen)) return new(RawSectorSize, Mode1DataOffset, dataLen); + if (TryProbe(image, RawSectorSize, Mode2Form1DataOffset, dataLen)) return new(RawSectorSize, Mode2Form1DataOffset, dataLen); + if (TryProbe(image, SectorSize2336, 8, dataLen)) return new(SectorSize2336, 8, dataLen); + if (TryProbe(image, Iso9660SectorSize, 0, dataLen)) return new(Iso9660SectorSize, 0, dataLen); + return new(RawSectorSize, Mode1DataOffset, dataLen); } - private static Trailer ReadTrailer(Stream image) { + private static long DetectDataAreaLength(Stream image) { + // Probe NER5 first (longer footer). if (image.Length >= Ner5FooterSize) { image.Position = image.Length - Ner5FooterSize; - Span footer = stackalloc byte[Ner5FooterSize]; - if (ReadExactly(image, footer) && footer[..4].SequenceEqual("NER5"u8)) { - var offset = BinaryPrimitives.ReadUInt64BigEndian(footer[4..]); - if (offset <= (ulong)(image.Length - Ner5FooterSize)) - return new(checked((long)offset), true); - } + Span tail = stackalloc byte[Ner5FooterSize]; + if (image.Read(tail) == Ner5FooterSize && + tail[0] == MagicNer5[0] && tail[1] == MagicNer5[1] && + tail[2] == MagicNer5[2] && tail[3] == MagicNer5[3]) + return image.Length - Ner5FooterSize; } - if (image.Length >= NeroFooterSize) { image.Position = image.Length - NeroFooterSize; - Span footer = stackalloc byte[NeroFooterSize]; - if (ReadExactly(image, footer) && footer[..4].SequenceEqual("NERO"u8)) { - var offset = BinaryPrimitives.ReadUInt32BigEndian(footer[4..]); - if (offset <= image.Length - NeroFooterSize) - return new(offset, true); - } + Span tail = stackalloc byte[NeroFooterSize]; + if (image.Read(tail) == NeroFooterSize && + tail[0] == MagicNero[0] && tail[1] == MagicNero[1] && + tail[2] == MagicNero[2] && tail[3] == MagicNero[3]) + return image.Length - NeroFooterSize; } - - return new(image.Length, false); + return image.Length; } private static bool TryProbe(Stream image, int sectorSize, int dataOffset, long dataAreaLength) { var pvdAt = (long)PvdLba * sectorSize + dataOffset; - if (pvdAt > dataAreaLength - 6) - return false; - + if (pvdAt + 6 > dataAreaLength) return false; image.Position = pvdAt; - Span signature = stackalloc byte[6]; - return ReadExactly(image, signature) && - signature[0] == 1 && signature[1..].SequenceEqual("CD001"u8); + Span sig = stackalloc byte[6]; + var read = image.Read(sig); + if (read < 6) return false; + return sig[0] == 1 && sig[1] == (byte)'C' && sig[2] == (byte)'D' && + sig[3] == (byte)'0' && sig[4] == (byte)'0' && sig[5] == (byte)'1'; } /// - /// Rewrites one existing 2,048-byte user-data sector. If the requested LBA - /// would extend a real NRG image, the operation is refused because moving the - /// descriptor requires rewriting ETN/DAO offsets and sizes as well. + /// Rewrites the 2 048-byte user-data region of sector + /// in place. Other bytes — sync/header/EDC for raw sectors, every other + /// sector, every other region of the image, and the trailing NRG footer — + /// are untouched. If points past the current + /// data-area EOF, the image is grown sector-by-sector with appended-sector + /// framing () and the footer is relocated to the + /// new EOF. /// + /// When .Length + /// differs from 2 048. public static void WriteSector(Stream image, int lba, ReadOnlySpan userData) { - var geometry = DetectGeometry(image); - WriteSector(image, lba, userData, geometry); + ArgumentNullException.ThrowIfNull(image); + if (lba < 0) throw new ArgumentOutOfRangeException(nameof(lba)); + if (userData.Length != Iso9660SectorSize) + throw new ArgumentException( + $"Sector user data must be exactly {Iso9660SectorSize} bytes; got {userData.Length}.", + nameof(userData)); + var geom = DetectGeometry(image); + WriteSector(image, lba, userData, geom); } - /// Rewrites a sector using an already detected geometry. - public static void WriteSector(Stream image, int lba, ReadOnlySpan userData, SectorGeometry geometry) { + /// + /// Variant of that + /// reuses a previously-probed geometry, avoiding a redundant PVD probe per + /// call when a caller is rewriting several sectors back-to-back. + /// + public static void WriteSector(Stream image, int lba, ReadOnlySpan userData, SectorGeometry geom) { ArgumentNullException.ThrowIfNull(image); - if (!image.CanWrite || !image.CanSeek) - throw new ArgumentException("NRG sector editing requires a writable, seekable stream.", nameof(image)); - if (lba < 0) - throw new ArgumentOutOfRangeException(nameof(lba)); + if (lba < 0) throw new ArgumentOutOfRangeException(nameof(lba)); if (userData.Length != Iso9660SectorSize) - throw new ArgumentException($"Sector user data must be exactly {Iso9660SectorSize} bytes; got {userData.Length}.", nameof(userData)); + throw new ArgumentException( + $"Sector user data must be exactly {Iso9660SectorSize} bytes; got {userData.Length}.", + nameof(userData)); - var sectorStart = checked((long)lba * geometry.SectorSize); - var sectorEnd = checked(sectorStart + geometry.SectorSize); - if (sectorEnd <= geometry.DataAreaLength) { - image.Position = checked(sectorStart + geometry.DataOffset); + var sectorStart = (long)lba * geom.SectorSize; + var endOfSector = sectorStart + geom.SectorSize; + + if (endOfSector <= geom.DataAreaLength) { + image.Position = sectorStart + geom.DataOffset; image.Write(userData); return; } - AppendSector(image, lba, userData, geometry); + AppendSector(image, lba, userData, geom); } /// - /// Appends sectors only to a footer-less raw image. Extending an NRG image is - /// not a byte-local operation because its descriptor records track offsets and - /// sizes; callers wanting growth must use the descriptor's rebuild editor. + /// Extends the data area so that sector exists, + /// writing as its 2 048-byte payload. + /// Intermediate sectors are appended with format-correct framing. The + /// trailing NRG footer is preserved verbatim and rewritten at the new EOF. /// - public static void AppendSector(Stream image, int lba, ReadOnlySpan userData, SectorGeometry geometry) { + public static void AppendSector(Stream image, int lba, ReadOnlySpan userData, SectorGeometry geom) { ArgumentNullException.ThrowIfNull(image); - if (lba < 0) - throw new ArgumentOutOfRangeException(nameof(lba)); + if (lba < 0) throw new ArgumentOutOfRangeException(nameof(lba)); if (userData.Length != Iso9660SectorSize) - throw new ArgumentException($"Sector user data must be exactly {Iso9660SectorSize} bytes; got {userData.Length}.", nameof(userData)); + throw new ArgumentException( + $"Sector user data must be exactly {Iso9660SectorSize} bytes; got {userData.Length}.", + nameof(userData)); + + byte[]? footer = null; + if (geom.DataAreaLength < image.Length) { + footer = new byte[image.Length - geom.DataAreaLength]; + image.Position = geom.DataAreaLength; + var got = 0; + while (got < footer.Length) { + var r = image.Read(footer, got, footer.Length - got); + if (r == 0) break; + got += r; + } + } - if (ReadTrailer(image).HasNrgFooter) - throw new NotSupportedException( - $"{Label}: extending a chunked image requires relocating and rewriting its track descriptor; use file-level rebuild modification instead."); + var firstMissingLba = (int)((geom.DataAreaLength + geom.SectorSize - 1) / geom.SectorSize); + var lastLbaToWrite = lba; - var firstMissingLba = checked((int)((geometry.DataAreaLength + geometry.SectorSize - 1) / geometry.SectorSize)); - for (var current = firstMissingLba; current < lba; ++current) { - image.Position = checked((long)current * geometry.SectorSize); - WriteFramedSector(image, geometry, ReadOnlySpan.Empty); + for (var i = firstMissingLba; i < lastLbaToWrite; i++) { + image.Position = (long)i * geom.SectorSize; + WriteFramedSector(image, geom, ReadOnlySpan.Empty); } - image.Position = checked((long)lba * geometry.SectorSize); - WriteFramedSector(image, geometry, userData); - image.SetLength(checked((long)(lba + 1) * geometry.SectorSize)); + image.Position = (long)lba * geom.SectorSize; + WriteFramedSector(image, geom, userData); + + if (footer != null) { + var newDataEnd = (long)(lba + 1) * geom.SectorSize; + image.SetLength(newDataEnd + footer.Length); + image.Position = newDataEnd; + image.Write(footer); + } else { + image.SetLength((long)(lba + 1) * geom.SectorSize); + } } - private static void WriteFramedSector(Stream image, SectorGeometry geometry, ReadOnlySpan userData) { - var sector = new byte[geometry.SectorSize]; - if (geometry.SectorSize == RawSectorSize) { - Sync.CopyTo(sector); - sector[15] = geometry.DataOffset == Mode2Form1DataOffset ? (byte)0x02 : (byte)0x01; + private static void WriteFramedSector(Stream image, SectorGeometry geom, ReadOnlySpan userData) { + var sector = new byte[geom.SectorSize]; + switch (geom.SectorSize) { + case RawSectorSize: + Sync.AsSpan().CopyTo(sector.AsSpan(0, 12)); + sector[15] = (byte)(geom.DataOffset == Mode2Form1DataOffset ? 0x02 : 0x01); + break; + case SectorSize2336: + break; } if (!userData.IsEmpty) - userData.CopyTo(sector.AsSpan(geometry.DataOffset, Iso9660SectorSize)); + userData.CopyTo(sector.AsSpan(geom.DataOffset, Iso9660SectorSize)); image.Write(sector); } - /// Zeros one existing sector's 2,048-byte user-data region. + /// + /// Zeros the 2 048-byte user-data region of sector + /// in place. The sector framing bytes and the trailing NRG footer are + /// preserved; only the user data is wiped. Returns true if the + /// sector existed (and was zeroed), false if + /// is past the data-area EOF. + /// public static bool ZeroSector(Stream image, int lba) { - if (lba < 0) - return false; - var geometry = DetectGeometry(image); - return ZeroSector(image, lba, geometry); + ArgumentNullException.ThrowIfNull(image); + if (lba < 0) return false; + var geom = DetectGeometry(image); + return ZeroSector(image, lba, geom); } - /// Zeros one existing sector using an already detected geometry. - public static bool ZeroSector(Stream image, int lba, SectorGeometry geometry) { + /// + /// Variant of reusing a previously-probed geometry. + /// + public static bool ZeroSector(Stream image, int lba, SectorGeometry geom) { ArgumentNullException.ThrowIfNull(image); - if (lba < 0) - return false; - var sectorEnd = checked((long)lba * geometry.SectorSize + geometry.SectorSize); - if (sectorEnd > geometry.DataAreaLength) - return false; - - Span zeros = stackalloc byte[Iso9660SectorSize]; - image.Position = checked((long)lba * geometry.SectorSize + geometry.DataOffset); + if (lba < 0) return false; + var endOfSector = (long)lba * geom.SectorSize + geom.SectorSize; + if (endOfSector > geom.DataAreaLength) return false; + var zeros = new byte[Iso9660SectorSize]; + image.Position = (long)lba * geom.SectorSize + geom.DataOffset; image.Write(zeros); return true; } - /// Parses a synthetic sector-NNNNNN.bin LBA name. + /// + /// Parses a synthetic sector-NNNNNN.bin entry name and returns the + /// embedded sector LBA. Names that don't match the schema return + /// false. + /// public static bool TryParseSectorEntryName(string entryName, out int lba) { lba = -1; - if (string.IsNullOrEmpty(entryName)) - return false; - + if (string.IsNullOrEmpty(entryName)) return false; var leaf = Path.GetFileName(entryName); const string prefix = "sector-"; const string suffix = ".bin"; - if (!leaf.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) || - !leaf.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) - return false; - + if (!leaf.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; + if (!leaf.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) return false; var numeric = leaf.AsSpan(prefix.Length, leaf.Length - prefix.Length - suffix.Length); return int.TryParse(numeric, NumberStyles.Integer, CultureInfo.InvariantCulture, out lba) && lba >= 0; } - /// Formats an LBA using the synthetic low-level sector namespace. + /// + /// Formats a sector LBA into the synthetic entry name used by the + /// in-place modifier. + /// public static string FormatSectorEntryName(int lba) => string.Create(CultureInfo.InvariantCulture, $"sector-{lba:D6}.bin"); - /// Rewrites sectors named through the synthetic low-level namespace. + /// + /// Routes each input through the sector-rewrite path. Inputs whose + /// ArchiveName matches sector-NNNNNN.bin are written at the + /// fixed LBA byte offset. Inputs whose ArchiveName doesn't match the + /// schema are refused — inner ISO 9660 directory mutation is + /// delegated to FileSystem.Iso. + /// public static void AddOrReplaceSectors(Stream image, IEnumerable<(string ArchiveName, byte[] Data)> inputs) { ArgumentNullException.ThrowIfNull(image); ArgumentNullException.ThrowIfNull(inputs); - var geometry = DetectGeometry(image); + var geom = DetectGeometry(image); foreach (var (name, data) in inputs) { + // A name this cannot place is not a name to pass over. These images list + // the ISO 9660 files inside them, so a caller has every reason to hand one + // back -- and skipping it wrote nothing, raised nothing, and reported the + // add as done. Six files added, six files gone, no error anywhere. if (!TryParseSectorEntryName(name, out var lba)) throw new NotSupportedException( - $"{Label}: '{name}' is not a fixed-LBA sector name. Use the descriptor's file-level modification path for ISO entries."); - WriteSector(image, lba, data, geometry); + $"{Label}: '{name}' cannot be added. This image is edited a sector at a time, so an " + + "entry has to be named 'sector-NNNN.bin' for the sector it replaces. Adding a file to " + + "the ISO 9660 filesystem inside the image is not something this supports."); + if (data.Length != Iso9660SectorSize) + throw new ArgumentException( + $"Sector entry '{name}' must carry exactly {Iso9660SectorSize} bytes; got {data.Length}.", + nameof(inputs)); + WriteSector(image, lba, data, geom); + geom = DetectGeometry(image); } } - /// Zeros sectors named through the synthetic low-level namespace. + /// + /// Zeros each named sector-NNNNNN.bin. Names that don't match the + /// schema are refused; sectors past the data-area EOF are still + /// skipped. The framing bytes of an existing sector and the trailing NRG + /// footer are preserved. + /// public static void RemoveSectors(Stream image, IEnumerable entryNames) { ArgumentNullException.ThrowIfNull(image); ArgumentNullException.ThrowIfNull(entryNames); - var geometry = DetectGeometry(image); + var geom = DetectGeometry(image); foreach (var name in entryNames) { if (!TryParseSectorEntryName(name, out var lba)) throw new NotSupportedException( - $"{Label}: '{name}' is not a fixed-LBA sector name. Use the descriptor's file-level modification path for ISO entries."); - _ = ZeroSector(image, lba, geometry); + $"{Label}: '{name}' cannot be removed. This image is edited a sector at a time, so an " + + "entry has to be named 'sector-NNNN.bin' for the sector it clears. Removing a file from " + + "the ISO 9660 filesystem inside the image is not something this supports."); + ZeroSector(image, lba, geom); } } - - private static bool ReadExactly(Stream stream, Span buffer) { - var offset = 0; - while (offset < buffer.Length) { - var read = stream.Read(buffer[offset..]); - if (read == 0) - return false; - offset += read; - } - return true; - } } diff --git a/FileFormats/FileFormat.Nrg/NrgReader.cs b/FileFormats/FileFormat.Nrg/NrgReader.cs index 41724ad30..9278ef0bb 100644 --- a/FileFormats/FileFormat.Nrg/NrgReader.cs +++ b/FileFormats/FileFormat.Nrg/NrgReader.cs @@ -1,32 +1,43 @@ -using System.Buffers.Binary; using System.Text; namespace FileFormat.Nrg; /// -/// Reads the ISO 9660 data track embedded in a Nero Burning ROM NRG image. -/// NRG stores disc sectors first, then a chunked session/track descriptor, and -/// finally a footer pointing back to that descriptor. +/// Reads the ISO 9660 file system embedded in a Nero Burning ROM NRG disc image. +/// NRG images carry a footer at the end of the file identifying the format version +/// and providing a chunk table that describes the track layout. +/// +/// Footer layout: +/// +/// NRG v2: last 12 bytes — "NER5" (4 bytes) + uint64 BE offset to chunk table. +/// NRG v1: last 8 bytes — "NERO" (4 bytes) + uint32 BE offset to chunk table. +/// +/// This reader parses the footer to locate the data area, then heuristically detects +/// the sector geometry and parses the ISO 9660 file system. +/// /// public sealed class NrgReader : IDisposable { + // Footer magic + private static readonly byte[] MagicNer5 = [(byte)'N', (byte)'E', (byte)'R', (byte)'5']; + private static readonly byte[] MagicNero = [(byte)'N', (byte)'E', (byte)'R', (byte)'O']; + + // ISO 9660 constants private const int Iso9660SectorSize = 2048; private const int RawSectorSize = 2352; private const int SectorSize2336 = 2336; - private const int RawSectorWithSubchannelSize = 2448; private const int PvdLba = 16; private const int Mode1DataOffset = 16; private const int Mode2Form1DataOffset = 24; private readonly Stream _stream; private readonly bool _leaveOpen; + private bool _disposed; + private readonly int _sectorSize; private readonly int _dataOffset; - private readonly long _trackOffset; - private readonly long _dataAreaEnd; - private bool _disposed; - private readonly record struct FooterInfo(int Version, long TrailerOffset, long FooterOffset); - private readonly record struct TrackCandidate(long Offset, long Length, int SectorSize, int DataOffset); + // Offset within the stream where the data (track) area begins + private readonly long _dataAreaOffset; /// Gets the NRG format version detected from the footer (1 or 2), or 0 if no valid footer was found. public int Version { get; } @@ -34,372 +45,279 @@ public sealed class NrgReader : IDisposable { /// Gets all file and directory entries found in the ISO 9660 file system. public IReadOnlyList Entries { get; } - /// Initializes a new from an NRG stream. + /// + /// Initializes a new from an NRG stream. + /// + /// The stream containing the NRG image data. + /// Whether to leave the stream open on dispose. public NrgReader(Stream stream, bool leaveOpen = false) { this._stream = stream ?? throw new ArgumentNullException(nameof(stream)); - if (!stream.CanRead || !stream.CanSeek) - throw new ArgumentException("NRG reading requires a readable, seekable stream.", nameof(stream)); this._leaveOpen = leaveOpen; - var footer = ReadFooter(stream); - this.Version = footer.Version; - - if (TryFindIsoTrack(stream, footer, out var track)) { - this._trackOffset = track.Offset; - this._sectorSize = track.SectorSize; - this._dataOffset = track.DataOffset; - this._dataAreaEnd = TrackEnd(track, footer.TrailerOffset); - } else { - var end = footer.Version != 0 ? footer.TrailerOffset : stream.Length; - (this._sectorSize, this._dataOffset) = DetectSectorGeometry(stream, 0, end); - this._trackOffset = 0; - this._dataAreaEnd = end; - } + (this.Version, this._dataAreaOffset) = ReadFooter(stream); + + // Treat the data area as a sub-stream starting at _dataAreaOffset + (this._sectorSize, this._dataOffset) = DetectSectorGeometry(stream, this._dataAreaOffset); var entries = new List(); TryParseIso9660(entries); this.Entries = entries; } - /// Extracts the raw data for a file entry. + /// + /// Extracts the raw data for a file entry. + /// + /// The file entry to extract. Must not be a directory. + /// The file data bytes. public byte[] Extract(NrgEntry entry) { ArgumentNullException.ThrowIfNull(entry); if (entry.IsDirectory) throw new ArgumentException("Cannot extract a directory entry.", nameof(entry)); if (entry.Size == 0) return []; - if (entry.Size > int.MaxValue) - throw new NotSupportedException("NRG entries larger than 2 GiB require a streaming extraction API."); - return ReadFileData(entry.StartLba, checked((int)entry.Size)); + return ReadFileData(entry.StartLba, (int)entry.Size); } - private static FooterInfo ReadFooter(Stream stream) { - if (stream.Length >= 12) { - stream.Position = stream.Length - 12; - Span footer = stackalloc byte[12]; - if (ReadExactly(stream, footer) && footer[..4].SequenceEqual("NER5"u8)) { - var offset = BinaryPrimitives.ReadUInt64BigEndian(footer[4..]); - var footerOffset = stream.Length - 12; - if (offset <= (ulong)footerOffset) - return new(2, checked((long)offset), footerOffset); - } + // ------------------------------------------------------------------------- + // Footer parsing + // ------------------------------------------------------------------------- + + private static (int Version, long DataAreaOffset) ReadFooter(Stream stream) { + if (stream.Length < 12) + return (0, 0); + + // Check NRG v2: last 12 bytes = "NER5" + 8-byte BE offset + stream.Position = stream.Length - 12; + Span footer12 = stackalloc byte[12]; + if (stream.Read(footer12) == 12 && + footer12[0] == MagicNer5[0] && footer12[1] == MagicNer5[1] && + footer12[2] == MagicNer5[2] && footer12[3] == MagicNer5[3]) { + var chunkOffset = ReadUInt64BE(footer12, 4); + // Data area starts at beginning of file; chunk table is at chunkOffset + // The data area offset is 0 (data precedes the chunk table) + return (2, 0); } + // Check NRG v1: last 8 bytes = "NERO" + 4-byte BE offset if (stream.Length >= 8) { stream.Position = stream.Length - 8; - Span footer = stackalloc byte[8]; - if (ReadExactly(stream, footer) && footer[..4].SequenceEqual("NERO"u8)) { - var offset = BinaryPrimitives.ReadUInt32BigEndian(footer[4..]); - var footerOffset = stream.Length - 8; - if (offset <= footerOffset) - return new(1, offset, footerOffset); - } - } - - return new(0, stream.Length, stream.Length); - } - - private static bool TryFindIsoTrack(Stream stream, FooterInfo footer, out TrackCandidate track) { - track = default; - if (footer.Version == 0 || footer.TrailerOffset >= footer.FooterOffset) - return false; - - foreach (var candidate in ReadTrackCandidates(stream, footer)) { - var end = TrackEnd(candidate, footer.TrailerOffset); - if (TryProbe(stream, candidate.Offset, candidate.SectorSize, candidate.DataOffset, end)) { - track = candidate; - return true; + Span footer8 = stackalloc byte[8]; + if (stream.Read(footer8) == 8 && + footer8[0] == MagicNero[0] && footer8[1] == MagicNero[1] && + footer8[2] == MagicNero[2] && footer8[3] == MagicNero[3]) { + return (1, 0); } } - return false; + // No recognized footer — treat entire stream as raw sector data starting at offset 0 + return (0, 0); } - private static List ReadTrackCandidates(Stream stream, FooterInfo footer) { - var result = new List(); - var position = footer.TrailerOffset; - var header = new byte[8]; + // ------------------------------------------------------------------------- + // Sector geometry detection + // ------------------------------------------------------------------------- - while (position <= footer.FooterOffset - header.Length) { - stream.Position = position; - if (!ReadExactly(stream, header)) - break; - - var headerSpan = header.AsSpan(); - var payloadLength = BinaryPrimitives.ReadUInt32BigEndian(headerSpan[4..]); - var payloadStart = position + header.Length; - if (payloadStart > footer.FooterOffset || payloadLength > (ulong)(footer.FooterOffset - payloadStart)) - break; - var payloadEnd = payloadStart + payloadLength; - - if (headerSpan[..4].SequenceEqual("ETN2"u8)) - ReadEtn2Candidates(stream, payloadStart, payloadLength, result); - else if (headerSpan[..4].SequenceEqual("ETNF"u8)) - ReadEtnfCandidates(stream, payloadStart, payloadLength, result); - else if (headerSpan[..4].SequenceEqual("DAOX"u8)) - ReadDaoCandidates(stream, payloadStart, payloadLength, isV2: true, result); - else if (headerSpan[..4].SequenceEqual("DAOI"u8)) - ReadDaoCandidates(stream, payloadStart, payloadLength, isV2: false, result); - - position = payloadEnd; - if (headerSpan[..4].SequenceEqual("END!"u8)) - break; - } - - return result; - } - - private static void ReadEtn2Candidates(Stream stream, long payloadStart, uint payloadLength, List result) { - const int recordSize = 32; - var record = new byte[recordSize]; - for (long relative = 0; relative + recordSize <= payloadLength; relative += recordSize) { - stream.Position = payloadStart + relative; - if (!ReadExactly(stream, record)) - return; - - var span = record.AsSpan(); - var offset = BinaryPrimitives.ReadUInt64BigEndian(span); - var length = BinaryPrimitives.ReadUInt64BigEndian(span[8..]); - if (offset > long.MaxValue || length > long.MaxValue) - continue; - if (TryDecodeMode(span[19], declaredSectorSize: 0, out var sectorSize, out var dataOffset)) - result.Add(new(checked((long)offset), checked((long)length), sectorSize, dataOffset)); - } - } - - private static void ReadEtnfCandidates(Stream stream, long payloadStart, uint payloadLength, List result) { - const int recordSize = 20; - var record = new byte[recordSize]; - for (long relative = 0; relative + recordSize <= payloadLength; relative += recordSize) { - stream.Position = payloadStart + relative; - if (!ReadExactly(stream, record)) - return; - - var span = record.AsSpan(); - var offset = BinaryPrimitives.ReadUInt32BigEndian(span); - var length = BinaryPrimitives.ReadUInt32BigEndian(span[4..]); - if (TryDecodeMode(span[11], declaredSectorSize: 0, out var sectorSize, out var dataOffset)) - result.Add(new(offset, length, sectorSize, dataOffset)); - } - } - - private static void ReadDaoCandidates(Stream stream, long payloadStart, uint payloadLength, bool isV2, List result) { - const int daoHeaderSize = 22; - var recordSize = isV2 ? 42 : 30; - if (payloadLength < daoHeaderSize) - return; - - var record = new byte[recordSize]; - for (long relative = daoHeaderSize; relative + recordSize <= payloadLength; relative += recordSize) { - stream.Position = payloadStart + relative; - if (!ReadExactly(stream, record)) - return; - - var span = record.AsSpan(); - var declaredSectorSize = BinaryPrimitives.ReadUInt16BigEndian(span.Slice(12, 2)); - var mode = span[14]; - ulong start; - ulong end; - if (isV2) { - start = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(26, 8)); - end = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(34, 8)); - } else { - start = BinaryPrimitives.ReadUInt32BigEndian(span.Slice(22, 4)); - end = BinaryPrimitives.ReadUInt32BigEndian(span.Slice(26, 4)); - } - - if (start > long.MaxValue || end > long.MaxValue || end < start) - continue; - if (TryDecodeMode(mode, declaredSectorSize, out var sectorSize, out var dataOffset)) - result.Add(new(checked((long)start), checked((long)(end - start)), sectorSize, dataOffset)); - } - } - - private static bool TryDecodeMode(byte mode, int declaredSectorSize, out int sectorSize, out int dataOffset) { - (sectorSize, dataOffset) = mode switch { - 0x00 or 0x02 => (Iso9660SectorSize, 0), - 0x03 => (SectorSize2336, 0), - 0x05 => (RawSectorSize, Mode1DataOffset), - 0x06 => (RawSectorSize, Mode2Form1DataOffset), - 0x0F => (RawSectorWithSubchannelSize, Mode1DataOffset), - 0x11 => (RawSectorWithSubchannelSize, Mode2Form1DataOffset), - _ => declaredSectorSize switch { - Iso9660SectorSize => (Iso9660SectorSize, 0), - _ => (0, 0), - }, - }; - return sectorSize != 0; - } - - private static long TrackEnd(TrackCandidate candidate, long hardEnd) { - if (candidate.Offset < 0 || candidate.Offset >= hardEnd) - return candidate.Offset; - if (candidate.Length <= 0) - return hardEnd; - var remaining = hardEnd - candidate.Offset; - return candidate.Offset + Math.Min(candidate.Length, remaining); - } - - private static (int SectorSize, int DataOffset) DetectSectorGeometry(Stream stream, long trackOffset, long dataEnd) { - if (TryProbe(stream, trackOffset, RawSectorSize, Mode1DataOffset, dataEnd)) + private static (int SectorSize, int DataOffset) DetectSectorGeometry(Stream stream, long dataAreaOffset) { + if (TryProbe(stream, dataAreaOffset, RawSectorSize, Mode1DataOffset)) return (RawSectorSize, Mode1DataOffset); - if (TryProbe(stream, trackOffset, RawSectorSize, Mode2Form1DataOffset, dataEnd)) + + if (TryProbe(stream, dataAreaOffset, RawSectorSize, Mode2Form1DataOffset)) return (RawSectorSize, Mode2Form1DataOffset); - if (TryProbe(stream, trackOffset, SectorSize2336, 8, dataEnd)) + + if (TryProbe(stream, dataAreaOffset, SectorSize2336, 8)) return (SectorSize2336, 8); - if (TryProbe(stream, trackOffset, Iso9660SectorSize, 0, dataEnd)) + + if (TryProbe(stream, dataAreaOffset, Iso9660SectorSize, 0)) return (Iso9660SectorSize, 0); + return (RawSectorSize, Mode1DataOffset); } - private static bool TryProbe(Stream stream, long trackOffset, int sectorSize, int dataOffset, long dataEnd) { - if (trackOffset < 0 || sectorSize <= 0 || dataOffset < 0) + private static bool TryProbe(Stream stream, long dataAreaOffset, int sectorSize, int dataOffset) { + var pvdPos = dataAreaOffset + (long)PvdLba * sectorSize + dataOffset; + if (pvdPos + 6 > stream.Length) return false; - var pvdPosition = trackOffset + (long)PvdLba * sectorSize + dataOffset; - if (pvdPosition < trackOffset || pvdPosition > dataEnd - 6) + + Span sig = stackalloc byte[6]; + stream.Position = pvdPos; + var read = stream.Read(sig); + if (read < 6) return false; - Span signature = stackalloc byte[6]; - stream.Position = pvdPosition; - return ReadExactly(stream, signature) && - signature[0] == 1 && signature[1..].SequenceEqual("CD001"u8); + return sig[0] == 1 && + sig[1] == (byte)'C' && + sig[2] == (byte)'D' && + sig[3] == (byte)'0' && + sig[4] == (byte)'0' && + sig[5] == (byte)'1'; } + // ------------------------------------------------------------------------- + // ISO 9660 parsing + // ------------------------------------------------------------------------- + private void TryParseIso9660(List entries) { var pvd = ReadSector(PvdLba); - if (pvd == null || pvd[0] != 1 || !pvd.AsSpan(1, 5).SequenceEqual("CD001"u8)) + if (pvd == null) return; - var rootLba = BinaryPrimitives.ReadUInt32LittleEndian(pvd.AsSpan(158, 4)); - var rootSize = BinaryPrimitives.ReadUInt32LittleEndian(pvd.AsSpan(166, 4)); - if (rootLba > int.MaxValue || rootSize > int.MaxValue) + if (pvd[0] != 1 || + pvd[1] != (byte)'C' || pvd[2] != (byte)'D' || + pvd[3] != (byte)'0' || pvd[4] != (byte)'0' || pvd[5] != (byte)'1') return; - WalkDirectory((int)rootLba, (int)rootSize, "", entries, []); + var rootLba = (int)ReadUInt32LE(pvd, 156 + 2); + var rootSize = (int)ReadUInt32LE(pvd, 156 + 10); + + WalkDirectory(rootLba, rootSize, "", entries); } - private void WalkDirectory(int dirLba, int dirSize, string parentPath, List entries, HashSet visited) { - if (dirLba <= 0 || dirSize <= 0 || !visited.Add(dirLba)) + private void WalkDirectory(int dirLba, int dirSize, string parentPath, List entries) { + if (dirLba <= 0 || dirSize <= 0) return; var bytesRead = 0; var currentLba = dirLba; - var bufferOffset = 0; + var bufOffset = 0; byte[]? sector = null; while (bytesRead < dirSize) { - if (sector == null || bufferOffset >= Iso9660SectorSize) { + if (sector == null || bufOffset >= Iso9660SectorSize) { sector = ReadSector(currentLba); if (sector == null) return; - ++currentLba; - bufferOffset = 0; + currentLba++; + bufOffset = 0; } - var recordLength = sector[bufferOffset]; - if (recordLength == 0) { - var remaining = Iso9660SectorSize - bufferOffset; + var recordLen = sector[bufOffset]; + + if (recordLen == 0) { + var remaining = Iso9660SectorSize - bufOffset; bytesRead += remaining; - bufferOffset = Iso9660SectorSize; + bufOffset = Iso9660SectorSize; continue; } - if (recordLength < 34 || bufferOffset + recordLength > Iso9660SectorSize) { - var remaining = Iso9660SectorSize - bufferOffset; + if (bufOffset + recordLen > Iso9660SectorSize) { + var remaining = Iso9660SectorSize - bufOffset; bytesRead += remaining; - bufferOffset = Iso9660SectorSize; + bufOffset = Iso9660SectorSize; continue; } - var record = sector.AsSpan(bufferOffset, recordLength); - ParseDirectoryRecord(record, parentPath, entries, visited); - bytesRead += recordLength; - bufferOffset += recordLength; + var record = sector.AsSpan(bufOffset, recordLen); + ParseDirectoryRecord(record, parentPath, entries); + + bytesRead += recordLen; + bufOffset += recordLen; } } - private void ParseDirectoryRecord(ReadOnlySpan record, string parentPath, List entries, HashSet visited) { - var idLength = record[32]; - if (idLength == 0 || 33 + idLength > record.Length) + private void ParseDirectoryRecord(ReadOnlySpan record, string parentPath, List entries) { + if (record.Length < 34) return; - if (idLength == 1 && (record[33] == 0x00 || record[33] == 0x01)) + + var dataLba = (int)ReadUInt32LE(record, 2); + var dataLen = (int)ReadUInt32LE(record, 10); + var flags = record[25]; + var idLen = record[32]; + + if (idLen == 0 || record.Length < 33 + idLen) return; - var dataLba = BinaryPrimitives.ReadUInt32LittleEndian(record[2..6]); - var dataLength = BinaryPrimitives.ReadUInt32LittleEndian(record[10..14]); - if (dataLba > int.MaxValue || dataLength > int.MaxValue) + if (idLen == 1 && (record[33] == 0x00 || record[33] == 0x01)) return; - var isDirectory = (record[25] & 0x02) != 0; - var rawName = Encoding.ASCII.GetString(record.Slice(33, idLength)); + var isDirectory = (flags & 0x02) != 0; + var rawName = Encoding.ASCII.GetString(record.Slice(33, idLen)); var name = StripVersionSuffix(rawName); + if (string.IsNullOrEmpty(name)) return; - var fullPath = parentPath.Length > 0 ? $"{parentPath}/{name}" : name; + var fullPath = parentPath.Length > 0 ? parentPath + "/" + name : name; + entries.Add(new NrgEntry { Name = name, FullPath = fullPath, IsDirectory = isDirectory, - Size = isDirectory ? 0 : dataLength, - StartLba = (int)dataLba, + Size = isDirectory ? 0 : dataLen, + StartLba = dataLba, }); if (isDirectory) - WalkDirectory((int)dataLba, (int)dataLength, fullPath, entries, visited); + WalkDirectory(dataLba, dataLen, fullPath, entries); } + // ------------------------------------------------------------------------- + // Data extraction + // ------------------------------------------------------------------------- + private byte[] ReadFileData(int startLba, int size) { var result = new byte[size]; var written = 0; - for (var lba = startLba; written < size; ++lba) { + var lba = startLba; + + while (written < size) { var sector = ReadSector(lba); if (sector == null) - throw new InvalidDataException("NRG file extent runs past the selected data track."); + break; var toCopy = Math.Min(Iso9660SectorSize, size - written); sector.AsSpan(0, toCopy).CopyTo(result.AsSpan(written)); written += toCopy; + lba++; } + return result; } private byte[]? ReadSector(int lba) { - if (lba < 0) - return null; - var sectorStart = this._trackOffset + (long)lba * this._sectorSize; + var sectorStart = this._dataAreaOffset + (long)lba * this._sectorSize; var dataStart = sectorStart + this._dataOffset; - if (sectorStart < this._trackOffset || dataStart < sectorStart || dataStart > this._dataAreaEnd - Iso9660SectorSize) + + if (dataStart + Iso9660SectorSize > this._stream.Length) return null; this._stream.Position = dataStart; - var buffer = new byte[Iso9660SectorSize]; - return ReadExactly(this._stream, buffer) ? buffer : null; - } - - private static bool ReadExactly(Stream stream, Span buffer) { - var offset = 0; - while (offset < buffer.Length) { - var read = stream.Read(buffer[offset..]); + var buf = new byte[Iso9660SectorSize]; + var totalRead = 0; + while (totalRead < buf.Length) { + var read = this._stream.Read(buf, totalRead, buf.Length - totalRead); if (read == 0) - return false; - offset += read; + return null; + totalRead += read; } - return true; + + return buf; } + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + private static uint ReadUInt32LE(ReadOnlySpan data, int offset) => + (uint)(data[offset] | (data[offset + 1] << 8) | (data[offset + 2] << 16) | (data[offset + 3] << 24)); + + private static ulong ReadUInt64BE(ReadOnlySpan data, int offset) => + ((ulong)data[offset] << 56) | ((ulong)data[offset + 1] << 48) | + ((ulong)data[offset + 2] << 40) | ((ulong)data[offset + 3] << 32) | + ((ulong)data[offset + 4] << 24) | ((ulong)data[offset + 5] << 16) | + ((ulong)data[offset + 6] << 8) | data[offset + 7]; + private static string StripVersionSuffix(string name) { - var separator = name.IndexOf(';'); - return separator >= 0 ? name[..separator] : name; + var semi = name.IndexOf(';'); + return semi >= 0 ? name[..semi] : name; } /// public void Dispose() { - if (this._disposed) - return; - this._disposed = true; - if (!this._leaveOpen) - this._stream.Dispose(); + if (!this._disposed) { + this._disposed = true; + if (!this._leaveOpen) + this._stream.Dispose(); + } } } diff --git a/FileFormats/FileFormat.Nrg/NrgStructureInspector.cs b/FileFormats/FileFormat.Nrg/NrgStructureInspector.cs deleted file mode 100644 index 127403774..000000000 --- a/FileFormats/FileFormat.Nrg/NrgStructureInspector.cs +++ /dev/null @@ -1,131 +0,0 @@ -using System.Buffers.Binary; - -namespace FileFormat.Nrg; - -/// Minimal descriptor-chain inspection shared by mutation/maintenance profile guards. -internal static class NrgStructureInspector { - internal readonly record struct Profile(int TrackCount, byte? SingleTrackMode) { - internal bool IsSingleDataTrack => this.TrackCount == 1 && this.SingleTrackMode is - 0x00 or 0x02 or 0x03 or 0x05 or 0x06 or 0x0F or 0x11; - } - - internal static Profile Inspect(Stream stream) { - ArgumentNullException.ThrowIfNull(stream); - if (!stream.CanRead || !stream.CanSeek) - throw new ArgumentException("NRG inspection requires a readable, seekable stream.", nameof(stream)); - - var saved = stream.Position; - try { - if (!TryGetDescriptorBounds(stream, out var descriptorOffset, out var footerOffset)) - return default; - - var trackCount = 0; - byte? singleMode = null; - var position = descriptorOffset; - Span header = stackalloc byte[8]; - while (position <= footerOffset - header.Length) { - stream.Position = position; - if (!ReadExactly(stream, header)) - break; - - var payloadLength = BinaryPrimitives.ReadUInt32BigEndian(header[4..]); - var payloadStart = position + header.Length; - if (payloadStart > footerOffset || payloadLength > (ulong)(footerOffset - payloadStart)) - break; - - if (header[..4].SequenceEqual("DAOX"u8)) - CountDao(stream, payloadStart, payloadLength, 42, ref trackCount, ref singleMode); - else if (header[..4].SequenceEqual("DAOI"u8)) - CountDao(stream, payloadStart, payloadLength, 30, ref trackCount, ref singleMode); - else if (header[..4].SequenceEqual("ETN2"u8)) - CountEtn(stream, payloadStart, payloadLength, 32, modeOffset: 19, ref trackCount, ref singleMode); - else if (header[..4].SequenceEqual("ETNF"u8)) - CountEtn(stream, payloadStart, payloadLength, 20, modeOffset: 11, ref trackCount, ref singleMode); - - position = payloadStart + payloadLength; - if (header[..4].SequenceEqual("END!"u8)) - break; - } - - return new(trackCount, trackCount == 1 ? singleMode : null); - } finally { - stream.Position = saved; - } - } - - private static void CountDao(Stream stream, long payloadStart, uint payloadLength, int recordSize, - ref int trackCount, ref byte? singleMode) { - const int headerSize = 22; - if (payloadLength < headerSize) - return; - var records = checked((int)((payloadLength - headerSize) / recordSize)); - for (var index = 0; index < records; ++index) { - stream.Position = payloadStart + headerSize + (long)index * recordSize + 14; - var mode = stream.ReadByte(); - if (mode < 0) - return; - AddMode((byte)mode, ref trackCount, ref singleMode); - } - } - - private static void CountEtn(Stream stream, long payloadStart, uint payloadLength, int recordSize, int modeOffset, - ref int trackCount, ref byte? singleMode) { - var records = checked((int)(payloadLength / recordSize)); - for (var index = 0; index < records; ++index) { - stream.Position = payloadStart + (long)index * recordSize + modeOffset; - var mode = stream.ReadByte(); - if (mode < 0) - return; - AddMode((byte)mode, ref trackCount, ref singleMode); - } - } - - private static void AddMode(byte mode, ref int trackCount, ref byte? singleMode) { - ++trackCount; - singleMode = trackCount == 1 ? mode : null; - } - - private static bool TryGetDescriptorBounds(Stream stream, out long descriptorOffset, out long footerOffset) { - descriptorOffset = 0; - footerOffset = 0; - - if (stream.Length >= 12) { - stream.Position = stream.Length - 12; - Span footer = stackalloc byte[12]; - if (ReadExactly(stream, footer) && footer[..4].SequenceEqual("NER5"u8)) { - var offset = BinaryPrimitives.ReadUInt64BigEndian(footer[4..]); - footerOffset = stream.Length - 12; - if (offset <= (ulong)footerOffset) { - descriptorOffset = checked((long)offset); - return true; - } - } - } - - if (stream.Length >= 8) { - stream.Position = stream.Length - 8; - Span footer = stackalloc byte[8]; - if (ReadExactly(stream, footer) && footer[..4].SequenceEqual("NERO"u8)) { - var offset = BinaryPrimitives.ReadUInt32BigEndian(footer[4..]); - footerOffset = stream.Length - 8; - if (offset <= footerOffset) { - descriptorOffset = offset; - return true; - } - } - } - - return false; - } - - private static bool ReadExactly(Stream stream, Span buffer) { - var offset = 0; - while (offset < buffer.Length) { - var read = stream.Read(buffer[offset..]); - if (read == 0) - return false; - offset += read; - } - return true; - } -} diff --git a/FileFormats/FileFormat.Nrg/NrgWriter.cs b/FileFormats/FileFormat.Nrg/NrgWriter.cs deleted file mode 100644 index 2dcc48e86..000000000 --- a/FileFormats/FileFormat.Nrg/NrgWriter.cs +++ /dev/null @@ -1,365 +0,0 @@ -using System.Buffers.Binary; -using System.Text; - -namespace FileFormat.Nrg; - -/// NRG v2 track storage modes understood by Nero-compatible readers. -public enum NrgTrackMode : byte { - /// Cooked Mode 1, 2,048 bytes per sector. - Mode1 = 0x00, - /// Cooked Mode 2 Form 1, 2,048 bytes per sector. - Mode2Form1 = 0x02, - /// Mode 2 Form 2 / almost-full sector, 2,336 bytes per sector. - Mode2Form2 = 0x03, - /// Raw Mode 1, 2,352 bytes per sector. - Mode1Raw = 0x05, - /// Raw Mode 2, 2,352 bytes per sector. - Mode2Raw = 0x06, - /// Raw CD-DA audio, 2,352 bytes per sector. - Audio = 0x07, - /// Raw Mode 1 plus 96-byte subchannel, 2,448 bytes per sector. - Mode1RawWithSubchannel = 0x0F, - /// Raw CD-DA audio plus 96-byte subchannel, 2,448 bytes per sector. - AudioWithSubchannel = 0x10, - /// Raw Mode 2 plus 96-byte subchannel, 2,448 bytes per sector. - Mode2RawWithSubchannel = 0x11, -} - -/// One track to be written into an NRG disc image. -/// On-disc/storage mode. -/// Track bytes, already encoded in the sector representation selected by . -public sealed record NrgTrackDefinition(NrgTrackMode Mode, byte[] Data) { - /// Number of sectors in index 00 before index 01. Defaults to no stored pregap. - public int PregapSectors { get; init; } - - /// - /// Optional pregap bytes. When supplied they must contain exactly sectors, - /// or, when is zero, determine the pregap length themselves. Raw framed - /// and subchannel modes require explicit bytes because an all-zero sector would not be valid framing. - /// - public byte[]? PregapData { get; init; } - - /// Optional 12-character ISRC for this track. - public string? Isrc { get; init; } - - /// Whether the Q-subchannel control byte advertises digital-copy permission. - public bool CopyPermitted { get; init; } -} - -/// One NRG session. Track numbers are assigned globally and consecutively across sessions. -/// Tracks in this session. -public sealed record NrgSessionDefinition(IReadOnlyList Tracks) { - /// Optional 13-digit media catalog number (MCN/EAN-13). - public string? Mcn { get; init; } -} - -/// A complete NRG disc authoring description. -/// Sessions in physical order. -public sealed record NrgDiscDefinition(IReadOnlyList Sessions) { - /// Optional raw CD-TEXT packs. The byte length must be a multiple of 18. - public byte[]? CdText { get; init; } -} - -/// -/// Writes Nero NRG v2 images using the DAO profile (CUEX + DAOX per session). -/// This profile can represent multiple sessions, mixed data/audio tracks, pregaps, MCN, ISRC, -/// raw sector modes and optional 96-byte subchannel data without flattening them into one ISO track. -/// -/// -/// Nero never published a normative NRG specification. The field layout used here is the independently -/// reconstructed public layout also consumed by CDEmu/libMirage and libcdio. No implementation code from -/// those projects is copied; this writer is an independent managed implementation of the observable format. -/// -public static class NrgWriter { - private const uint CdRomMediumType = 0x00000400; - private const int LeadInSectors = 4500; - private const int FirstLeadOutSectors = 6750; - private const int LaterLeadOutSectors = 2250; - - private sealed record TrackLayout( - int TrackNumber, - NrgTrackDefinition Definition, - int SectorSize, - int PregapSectors, - long PregapOffset, - long StartOffset, - long EndOffset, - int Index0Lba, - int Index1Lba); - - private sealed record SessionLayout( - NrgSessionDefinition Definition, - int LeadInLba, - int LeadOutLba, - IReadOnlyList Tracks); - - /// Writes as an NRG v2 image. - public static void Write(Stream output, NrgDiscDefinition disc) { - ArgumentNullException.ThrowIfNull(output); - ArgumentNullException.ThrowIfNull(disc); - if (!output.CanWrite || !output.CanSeek) - throw new ArgumentException("NRG authoring requires a writable, seekable stream.", nameof(output)); - - ValidateDisc(disc); - output.Position = 0; - output.SetLength(0); - - var sessions = WriteTrackData(output, disc); - var trailerOffset = checked((ulong)output.Position); - - // Allocated once, outside the loop: a stackalloc per iteration grows the frame with the - // session count (CA2014). The four bytes are fully rewritten each time. - Span sinf = stackalloc byte[4]; - foreach (var session in sessions) { - WriteCuex(output, session); - WriteDaox(output, session); - - BinaryPrimitives.WriteUInt32BigEndian(sinf, checked((uint)session.Tracks.Count)); - WriteChunk(output, "SINF"u8, sinf); - } - - if (disc.CdText is { Length: > 0 } cdText) - WriteChunk(output, "CDTX"u8, cdText); - - Span mtyp = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32BigEndian(mtyp, CdRomMediumType); - WriteChunk(output, "MTYP"u8, mtyp); - WriteChunk(output, "END!"u8, ReadOnlySpan.Empty); - - Span footer = stackalloc byte[12]; - "NER5"u8.CopyTo(footer); - BinaryPrimitives.WriteUInt64BigEndian(footer[4..], trailerOffset); - output.Write(footer); - } - - /// Gets the stored sector size implied by an NRG mode code. - public static int GetSectorSize(NrgTrackMode mode) => mode switch { - NrgTrackMode.Mode1 or NrgTrackMode.Mode2Form1 => 2048, - NrgTrackMode.Mode2Form2 => 2336, - NrgTrackMode.Mode1Raw or NrgTrackMode.Mode2Raw or NrgTrackMode.Audio => 2352, - NrgTrackMode.Mode1RawWithSubchannel or NrgTrackMode.AudioWithSubchannel or NrgTrackMode.Mode2RawWithSubchannel => 2448, - _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, "Unsupported NRG track mode."), - }; - - private static List WriteTrackData(Stream output, NrgDiscDefinition disc) { - var sessions = new List(disc.Sessions.Count); - var nextTrackNumber = 1; - var previousLeadOutLba = 0; - - for (var sessionIndex = 0; sessionIndex < disc.Sessions.Count; ++sessionIndex) { - var definition = disc.Sessions[sessionIndex]; - var firstTrackPregap = GetPregapSectors(definition.Tracks[0], GetSectorSize(definition.Tracks[0].Mode)); - int leadInLba; - int programLba; - if (sessionIndex == 0) { - leadInLba = -150; - programLba = -firstTrackPregap; - } else { - var previousLeadOutLength = sessionIndex == 1 ? FirstLeadOutSectors : LaterLeadOutSectors; - leadInLba = checked(previousLeadOutLba + previousLeadOutLength); - programLba = checked(leadInLba + LeadInSectors); - } - - var tracks = new List(definition.Tracks.Count); - foreach (var track in definition.Tracks) { - var sectorSize = GetSectorSize(track.Mode); - var pregapSectors = GetPregapSectors(track, sectorSize); - var pregapOffset = output.Position; - if (pregapSectors != 0) { - if (track.PregapData is { Length: > 0 } pregapData) - output.Write(pregapData); - else - WriteZeros(output, checked((long)pregapSectors * sectorSize)); - } - - var startOffset = output.Position; - output.Write(track.Data); - var endOffset = output.Position; - var index0Lba = programLba; - var index1Lba = checked(index0Lba + pregapSectors); - var dataSectors = track.Data.Length / sectorSize; - programLba = checked(index1Lba + dataSectors); - - tracks.Add(new TrackLayout( - nextTrackNumber++, track, sectorSize, pregapSectors, - pregapOffset, startOffset, endOffset, index0Lba, index1Lba)); - } - - var leadOutLba = programLba; - sessions.Add(new SessionLayout(definition, leadInLba, leadOutLba, tracks)); - previousLeadOutLba = leadOutLba; - } - - return sessions; - } - - private static void WriteCuex(Stream output, SessionLayout session) { - var payload = new byte[checked((session.Tracks.Count * 2 + 2) * 8)]; - var position = 0; - - var firstAdrCtl = AdrCtl(session.Tracks[0].Definition); - WriteCueEntry(payload.AsSpan(position, 8), firstAdrCtl, 0x00, 0x00, session.LeadInLba); - position += 8; - - foreach (var track in session.Tracks) { - var adrCtl = AdrCtl(track.Definition); - var trackBcd = ToBcd(track.TrackNumber); - WriteCueEntry(payload.AsSpan(position, 8), adrCtl, trackBcd, 0x00, track.Index0Lba); - position += 8; - WriteCueEntry(payload.AsSpan(position, 8), adrCtl, trackBcd, 0x01, track.Index1Lba); - position += 8; - } - - var lastAdrCtl = AdrCtl(session.Tracks[^1].Definition); - WriteCueEntry(payload.AsSpan(position, 8), lastAdrCtl, 0xAA, 0x01, session.LeadOutLba); - WriteChunk(output, "CUEX"u8, payload); - } - - private static void WriteDaox(Stream output, SessionLayout session) { - var payload = new byte[checked(22 + session.Tracks.Count * 42)]; - BinaryPrimitives.WriteUInt32BigEndian(payload, checked((uint)payload.Length)); - - if (session.Definition.Mcn is { } mcn) - Encoding.ASCII.GetBytes(mcn, payload.AsSpan(4, 13)); - - payload[17] = 0; - BinaryPrimitives.WriteUInt16BigEndian(payload.AsSpan(18, 2), GetTocType(session)); - payload[20] = checked((byte)session.Tracks[0].TrackNumber); - payload[21] = checked((byte)session.Tracks[^1].TrackNumber); - - var position = 22; - foreach (var track in session.Tracks) { - var record = payload.AsSpan(position, 42); - if (track.Definition.Isrc is { } isrc) - Encoding.ASCII.GetBytes(isrc, record[..12]); - BinaryPrimitives.WriteUInt16BigEndian(record[12..], checked((ushort)track.SectorSize)); - record[14] = (byte)track.Definition.Mode; - record[15] = 0; - BinaryPrimitives.WriteUInt16BigEndian(record[16..], 1); - BinaryPrimitives.WriteUInt64BigEndian(record[18..], checked((ulong)track.PregapOffset)); - BinaryPrimitives.WriteUInt64BigEndian(record[26..], checked((ulong)track.StartOffset)); - BinaryPrimitives.WriteUInt64BigEndian(record[34..], checked((ulong)track.EndOffset)); - position += 42; - } - - WriteChunk(output, "DAOX"u8, payload); - } - - private static ushort GetTocType(SessionLayout session) { - if (session.Tracks.Any(static track => IsMode2(track.Definition.Mode))) - return 0x2001; - return session.Tracks.Any(static track => !IsAudio(track.Definition.Mode)) ? (ushort)0x0001 : (ushort)0x0000; - } - - private static void WriteCueEntry(Span destination, byte adrCtl, byte track, byte index, int lba) { - destination[0] = adrCtl; - destination[1] = track; - destination[2] = index; - destination[3] = 0; - BinaryPrimitives.WriteUInt32BigEndian(destination[4..], unchecked((uint)lba)); - } - - private static byte AdrCtl(NrgTrackDefinition track) { - var control = IsAudio(track.Mode) ? 0x00 : 0x40; - if (track.CopyPermitted) - control |= 0x20; - return checked((byte)(control | 0x01)); - } - - private static byte ToBcd(int value) { - if (value is < 0 or > 99) - throw new ArgumentOutOfRangeException(nameof(value), value, "BCD track numbers are limited to 0..99."); - return checked((byte)(((value / 10) << 4) | value % 10)); - } - - private static int GetPregapSectors(NrgTrackDefinition track, int sectorSize) { - if (track.PregapSectors < 0) - throw new ArgumentOutOfRangeException(nameof(track.PregapSectors), "Pregap sector count cannot be negative."); - if (track.PregapData is not { Length: > 0 } bytes) { - if (track.PregapSectors != 0 && RequiresExplicitPregapData(track.Mode)) - throw new ArgumentException($"NRG {track.Mode} pregaps require explicit PregapData with valid sector framing.", nameof(track)); - return track.PregapSectors; - } - if (bytes.Length % sectorSize != 0) - throw new ArgumentException("NRG pregap data must be an integral number of sectors.", nameof(track)); - - var fromBytes = bytes.Length / sectorSize; - if (track.PregapSectors != 0 && track.PregapSectors != fromBytes) - throw new ArgumentException("NRG PregapSectors does not match PregapData length.", nameof(track)); - return fromBytes; - } - - private static bool RequiresExplicitPregapData(NrgTrackMode mode) - => mode is NrgTrackMode.Mode1Raw or NrgTrackMode.Mode2Raw or - NrgTrackMode.Mode1RawWithSubchannel or NrgTrackMode.AudioWithSubchannel or NrgTrackMode.Mode2RawWithSubchannel; - - private static void ValidateDisc(NrgDiscDefinition disc) { - if (disc.Sessions is null || disc.Sessions.Count == 0) - throw new ArgumentException("An NRG disc must contain at least one session.", nameof(disc)); - if (disc.CdText is { Length: > 0 } cdText && cdText.Length % 18 != 0) - throw new ArgumentException("NRG CD-TEXT data must contain complete 18-byte packs.", nameof(disc)); - - var trackCount = 0; - foreach (var session in disc.Sessions) { - if (session is null || session.Tracks is null || session.Tracks.Count == 0) - throw new ArgumentException("Every NRG session must contain at least one track.", nameof(disc)); - ValidateMcn(session.Mcn); - - foreach (var track in session.Tracks) { - if (track is null) - throw new ArgumentException("NRG tracks cannot be null.", nameof(disc)); - var sectorSize = GetSectorSize(track.Mode); - if (track.Data is null || track.Data.Length == 0 || track.Data.Length % sectorSize != 0) - throw new ArgumentException($"NRG {track.Mode} track data must be a non-empty multiple of {sectorSize} bytes.", nameof(disc)); - _ = GetPregapSectors(track, sectorSize); - ValidateIsrc(track.Isrc); - } - - trackCount = checked(trackCount + session.Tracks.Count); - } - - if (trackCount > 99) - throw new ArgumentException("Compact Disc NRG images can contain at most 99 tracks.", nameof(disc)); - } - - private static void ValidateMcn(string? mcn) { - if (mcn is null) - return; - if (mcn.Length != 13 || mcn.Any(static c => c is < '0' or > '9')) - throw new ArgumentException("NRG MCN must contain exactly 13 ASCII digits.", nameof(mcn)); - } - - private static void ValidateIsrc(string? isrc) { - if (isrc is null) - return; - if (isrc.Length != 12 || isrc.Any(static c => c > 0x7F || char.IsControl(c))) - throw new ArgumentException("NRG ISRC must contain exactly 12 printable ASCII characters.", nameof(isrc)); - } - - private static bool IsAudio(NrgTrackMode mode) - => mode is NrgTrackMode.Audio or NrgTrackMode.AudioWithSubchannel; - - private static bool IsMode2(NrgTrackMode mode) - => mode is NrgTrackMode.Mode2Form1 or NrgTrackMode.Mode2Form2 or NrgTrackMode.Mode2Raw or NrgTrackMode.Mode2RawWithSubchannel; - - private static void WriteZeros(Stream output, long length) { - Span zeros = stackalloc byte[4096]; - zeros.Clear(); - while (length > 0) { - var count = (int)Math.Min(length, zeros.Length); - output.Write(zeros[..count]); - length -= count; - } - } - - private static void WriteChunk(Stream output, ReadOnlySpan id, ReadOnlySpan payload) { - if (id.Length != 4) - throw new ArgumentException("NRG chunk ids are exactly four bytes.", nameof(id)); - - Span header = stackalloc byte[8]; - id.CopyTo(header); - BinaryPrimitives.WriteUInt32BigEndian(header[4..], checked((uint)payload.Length)); - output.Write(header); - output.Write(payload); - } -} From 79ade68c961289a428aff6b1b635fb606f96f168 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 13:43:29 +0200 Subject: [PATCH 164/165] * regenerate the filesystem support matrix after the branch drops --- Hawkynt.FileFormats.FileSystems/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index b9f7f63b0..39bbe9968 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -67,7 +67,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Intel HEX](https://en.wikipedia.org/wiki/Intel_HEX) | `IntelHex` | R/W | ✅ | ✅ rebuild | — | — | — | ✅ | own reader + struct-parity tests | Intel HEX and Motorola S-record, normalised to `firmware.bin` + metadata; the writer takes one payload and the base and start addresses from `metadata.ini` | [Intel HEX](https://www.keil.com/support/docs/1584/_hlp_hexfile.htm) / [S-record](https://srecord.sourceforge.net/man/man5/srec_motorola.5.html) | | [MDF/MDS](https://en.wikipedia.org/wiki/Alcohol_120%25) | `Mdf` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Alcohol 120%](https://en.wikipedia.org/wiki/Alcohol_120%25) | | [Nintendo PartitionFS](https://en.wikipedia.org/wiki/Nintendo_Switch) | `Pfs0` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | Nintendo Switch PartitionFS | [PFS0 (switchbrew)](https://switchbrew.org/wiki/NCA#PFS0) | -| [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | ✅ | ✅ rebuild | — | ✅ | — | ✅ | own reader + struct-parity tests | NRG v1/v2 reader; v2 TAO writer; named edits rebuild the embedded ISO | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | +| [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | NRG v1/v2 reader; v2 TAO writer; named edits rebuild the embedded ISO | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | | [PSP CSO/ZSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | `Cso` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Compressed ISO; edits re-pack the affected blocks | [CSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | | [QCOW2](https://en.wikipedia.org/wiki/Qcow) | `Qcow2` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | [`qemu-img check`](https://www.qemu.org/docs/master/tools/qemu-img.html) / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [QCOW2 specification](https://www.qemu.org/docs/master/interop/qcow2.html) | | [T64](https://en.wikipedia.org/wiki/Commodore_64) | `T64` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 tape archive | [T64 (VICE)](https://vice-emu.sourceforge.io/vice_17.html) | From 4bfd004428a4faa7c24c962ce5e305e8d335ff83 Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Fri, 11 Sep 2026 14:24:34 +0200 Subject: [PATCH 165/165] * regenerate the package API references for the integrated branches The fourteen branches changed the public surface of Compression.Core, FileFormats.Archives and FileFormats.FileSystems. --- Compression.Core/README.md | 2 +- Compression.Core/REFERENCE.md | 14 +- Hawkynt.FileFormats.Archives/REFERENCE.md | 11 +- Hawkynt.FileFormats.FileSystems/README.md | 4 +- Hawkynt.FileFormats.FileSystems/REFERENCE.md | 473 +++++++++++-------- 5 files changed, 311 insertions(+), 193 deletions(-) diff --git a/Compression.Core/README.md b/Compression.Core/README.md index a5c5465c1..82ec5cb56 100644 --- a/Compression.Core/README.md +++ b/Compression.Core/README.md @@ -204,7 +204,7 @@ Use the concrete version you intend to consume; this document does not predict a -Every public and protected member of all 627 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Compression.Core/REFERENCE.md). +Every public and protected member of all 628 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Compression.Core/REFERENCE.md). diff --git a/Compression.Core/REFERENCE.md b/Compression.Core/REFERENCE.md index 1f7348efc..b5c55a7ca 100644 --- a/Compression.Core/REFERENCE.md +++ b/Compression.Core/REFERENCE.md @@ -6295,7 +6295,7 @@ Run-Length Encoding (RLE) transform. Encodes runs of identical bytes as (count, ### Namespace `Compression.Registry` -[`AlgorithmFamily`](#algorithmfamily) · [`ApeTagReader`](#apetagreader) · [`ApeTagReader.ApeTag`](#apetagreaderapetag) · [`ArchiveEntryInfo`](#archiveentryinfo) · [`ArchiveInputInfo`](#archiveinputinfo) · [`ArchiveShrinker`](#archiveshrinker) · [`AudioEncodedStream`](#audioencodedstream) · [`AudioPacket`](#audiopacket) · [`AudioPcmBuffer`](#audiopcmbuffer) · [`AudioPcmEncoding`](#audiopcmencoding) · [`AudioPcmFormat`](#audiopcmformat) · [`AudioPseudoArchive`](#audiopseudoarchive) · [`AudioPseudoArchive.Entry`](#audiopseudoarchiveentry) · [`AudioStreamFormat`](#audiostreamformat) · [`BlockDeviceGeometry`](#blockdevicegeometry) · [`BlockDeviceStream`](#blockdevicestream) · [`BuildingBlockRegistry`](#buildingblockregistry) · [`CompoundTarDescriptor`](#compoundtardescriptor) · [`DefragBlockClass`](#defragblockclass) · [`DefragBlockInfo`](#defragblockinfo) · [`DefragBlockKind`](#defragblockkind) · [`DefragContentGuard`](#defragcontentguard) · [`DefragMode`](#defragmode) · [`DefragOptions`](#defragoptions) · [`DefragProgressEvent`](#defragprogressevent) · [`DefragRebuilder`](#defragrebuilder) · [`EntropyDetector`](#entropydetector) · [`FatDirStamp`](#fatdirstamp) · [`FilesystemCompressionParameter`](#filesystemcompressionparameter) · [`FilesystemCompressionProfile`](#filesystemcompressionprofile) · [`FilesystemDirectoryEntry`](#filesystemdirectoryentry) · [`FilesystemDriverBindingKind`](#filesystemdriverbindingkind) · [`FilesystemDriverCapabilities`](#filesystemdrivercapabilities) · [`FilesystemDriverCoverage`](#filesystemdrivercoverage) · [`FilesystemDriverDerivation`](#filesystemdriverderivation) · [`FilesystemDriverProfile`](#filesystemdriverprofile) · [`FilesystemDriverReadinessLayer`](#filesystemdriverreadinesslayer) · [`FilesystemDriverReadinessReport`](#filesystemdriverreadinessreport) · [`FilesystemDriverTarget`](#filesystemdrivertarget) · [`FilesystemMetadataPatch`](#filesystemmetadatapatch) · [`FilesystemMutationModel`](#filesystemmutationmodel) · [`FilesystemNodeId`](#filesystemnodeid) · [`FilesystemNodeInfo`](#filesystemnodeinfo) · [`FilesystemNodeKind`](#filesystemnodekind) · [`FilesystemOpenOptions`](#filesystemopenoptions) · [`FilesystemOptimization`](#filesystemoptimization) · [`FilesystemOptimizationAdapters`](#filesystemoptimizationadapters) · [`FilesystemOptimizationAdapters.HardLinkDeduplicator`](#filesystemoptimizationadaptershardlinkdeduplicator) · [`FilesystemOptimizationAdapters.SymbolicLinkDeduplicator`](#filesystemoptimizationadapterssymboliclinkdeduplicator) · [`FilesystemOptimizationExtensions`](#filesystemoptimizationextensions) · [`FilesystemOptimizationFeatures`](#filesystemoptimizationfeatures) · [`FilesystemOptimizationOptions`](#filesystemoptimizationoptions) · [`FilesystemSchemaPresets`](#filesystemschemapresets) · [`FilesystemSnapshotDirectoryEntry`](#filesystemsnapshotdirectoryentry) · [`FilesystemSnapshotNode`](#filesystemsnapshotnode) · [`FormatCapabilities`](#formatcapabilities) · [`FormatCategory`](#formatcategory) · [`FormatCreateOptions`](#formatcreateoptions) · [`FormatDetectionSignature`](#formatdetectionsignature) · [`FormatHeaderMatch`](#formatheadermatch) · [`FormatHealth`](#formathealth) · [`FormatHelpers`](#formathelpers) · [`FormatMethodInfo`](#formatmethodinfo) · [`FormatOptionDescriptor`](#formatoptiondescriptor) · [`FormatOptionKind`](#formatoptionkind) · [`FormatRegistry`](#formatregistry) · [`HardLinkDeduplicationSemantics`](#hardlinkdeduplicationsemantics) · [`IArchiveCreatable`](#iarchivecreatable) · [`IArchiveDefragmentable`](#iarchivedefragmentable) · [`IArchiveFormatOperations`](#iarchiveformatoperations) · [`IArchiveInMemoryExtract`](#iarchiveinmemoryextract) · [`IArchiveLayoutMap`](#iarchivelayoutmap) · [`IArchiveModifiable`](#iarchivemodifiable) · [`IArchivePurgeable`](#iarchivepurgeable) · [`IArchiveShrinkable`](#iarchiveshrinkable) · [`IArchiveWriteConstraints`](#iarchivewriteconstraints) · [`IAsyncArchiveOperations`](#iasyncarchiveoperations) · [`IAudioContainerFormat`](#iaudiocontainerformat) · [`IAudioDemuxSource`](#iaudiodemuxsource) · [`IAudioMuxTarget`](#iaudiomuxtarget) · [`IAudioPcmSource`](#iaudiopcmsource) · [`IAudioPcmTarget`](#iaudiopcmtarget) · [`IBlockDeviceFilesystemDriverProvider`](#iblockdevicefilesystemdriverprovider) · [`IBlockDeviceProvider`](#iblockdeviceprovider) · [`IBuildingBlock`](#ibuildingblock) · [`IContainerRemuxable`](#icontainerremuxable) · [`IFileInternalChunkMover`](#ifileinternalchunkmover) · [`IFileInternalLayoutMap`](#ifileinternallayoutmap) · [`IFilesystemBlockMover`](#ifilesystemblockmover) · [`IFilesystemDriverAdapter`](#ifilesystemdriveradapter) · [`IFilesystemDriverProvider`](#ifilesystemdriverprovider) · [`IFilesystemDriverReadinessProvider`](#ifilesystemdriverreadinessprovider) · [`IFilesystemExtentMap`](#ifilesystemextentmap) · [`IFilesystemFileHandle`](#ifilesystemfilehandle) · [`IFilesystemMetadataMover`](#ifilesystemmetadatamover) · [`IFilesystemPlaceable`](#ifilesystemplaceable) · [`IFilesystemScrambleable`](#ifilesystemscrambleable) · [`IFilesystemSession`](#ifilesystemsession) · [`IFilesystemTransaction`](#ifilesystemtransaction) · [`IFormatDescriptor`](#iformatdescriptor) · [`IFormatDetectionSource`](#iformatdetectionsource) · [`IFormatOptionsSchema`](#iformatoptionsschema) · [`IFormatValidator`](#iformatvalidator) · [`ILayoutOptimizable`](#ilayoutoptimizable) · [`IPartitionEditable`](#ipartitioneditable) · [`IRandomAccessBlockDevice`](#irandomaccessblockdevice) · [`IRandomAccessBlockDeviceProvider`](#irandomaccessblockdeviceprovider) · [`IRawTrackDevice`](#irawtrackdevice) · [`IRawTrackDeviceProvider`](#irawtrackdeviceprovider) · [`IStreamFormatOperations`](#istreamformatoperations) · [`ISymbolicLinkDeduplicationLayout`](#isymboliclinkdeduplicationlayout) · [`ISyntheticEntryNames`](#isyntheticentrynames) · [`IWipeEmpty`](#iwipeempty) · [`InnerFsDetector`](#innerfsdetector) · [`IssueSeverity`](#issueseverity) · [`LayoutAnalysis`](#layoutanalysis) · [`LayoutPatch`](#layoutpatch) · [`LayoutProfile`](#layoutprofile) · [`LayoutRebuildOptions`](#layoutrebuildoptions) · [`LayoutReclaim`](#layoutreclaim) · [`MagicSignature`](#magicsignature) · [`MediaProfile`](#mediaprofile) · [`MediaProfileLookup`](#mediaprofilelookup) · [`MetadataPlacementProfile`](#metadataplacementprofile) · [`MetadataPlacementRule`](#metadataplacementrule) · [`MetadataZone`](#metadatazone) · [`MethodNameParser`](#methodnameparser) · [`ModifyRebuilder`](#modifyrebuilder) · [`MutableRebuildFilesystemSession`](#mutablerebuildfilesystemsession) · [`MutableRebuildFilesystemSession.RebuildImage`](#mutablerebuildfilesystemsessionrebuildimage) · [`MutableRebuildFilesystemSession.ValidateImage`](#mutablerebuildfilesystemsessionvalidateimage) · [`PartitionBlockDevice`](#partitionblockdevice) · [`PlacementOptions`](#placementoptions) · [`PlacementZone`](#placementzone) · [`RawDiskShrinkRebuilder`](#rawdiskshrinkrebuilder) · [`RawTrackInfo`](#rawtrackinfo) · [`ReadOnlyFilesystemSnapshotSession`](#readonlyfilesystemsnapshotsession) · [`RebuildFilesystemEntry`](#rebuildfilesystementry) · [`RebuildVerb`](#rebuildverb) · [`ScrambleOptions`](#scrambleoptions) · [`SpoolingReadOnlyFileHandle`](#spoolingreadonlyfilehandle) · [`StreamBlockDevice`](#streamblockdevice) · [`SymlinkResolver`](#symlinkresolver) · [`UnusedSpaceWiper`](#unusedspacewiper) · [`ValidationIssue`](#validationissue) · [`ValidationLevel`](#validationlevel) · [`ValidationResult`](#validationresult) · [`WholeImageRebuildCommitter`](#wholeimagerebuildcommitter) +[`AlgorithmFamily`](#algorithmfamily) · [`ApeTagReader`](#apetagreader) · [`ApeTagReader.ApeTag`](#apetagreaderapetag) · [`ArchiveEntryInfo`](#archiveentryinfo) · [`ArchiveInputInfo`](#archiveinputinfo) · [`ArchiveMutationOptions`](#archivemutationoptions) · [`ArchiveShrinker`](#archiveshrinker) · [`AudioEncodedStream`](#audioencodedstream) · [`AudioPacket`](#audiopacket) · [`AudioPcmBuffer`](#audiopcmbuffer) · [`AudioPcmEncoding`](#audiopcmencoding) · [`AudioPcmFormat`](#audiopcmformat) · [`AudioPseudoArchive`](#audiopseudoarchive) · [`AudioPseudoArchive.Entry`](#audiopseudoarchiveentry) · [`AudioStreamFormat`](#audiostreamformat) · [`BlockDeviceGeometry`](#blockdevicegeometry) · [`BlockDeviceStream`](#blockdevicestream) · [`BuildingBlockRegistry`](#buildingblockregistry) · [`CompoundTarDescriptor`](#compoundtardescriptor) · [`DefragBlockClass`](#defragblockclass) · [`DefragBlockInfo`](#defragblockinfo) · [`DefragBlockKind`](#defragblockkind) · [`DefragContentGuard`](#defragcontentguard) · [`DefragMode`](#defragmode) · [`DefragOptions`](#defragoptions) · [`DefragProgressEvent`](#defragprogressevent) · [`DefragRebuilder`](#defragrebuilder) · [`EntropyDetector`](#entropydetector) · [`FatDirStamp`](#fatdirstamp) · [`FilesystemCompressionParameter`](#filesystemcompressionparameter) · [`FilesystemCompressionProfile`](#filesystemcompressionprofile) · [`FilesystemDirectoryEntry`](#filesystemdirectoryentry) · [`FilesystemDriverBindingKind`](#filesystemdriverbindingkind) · [`FilesystemDriverCapabilities`](#filesystemdrivercapabilities) · [`FilesystemDriverCoverage`](#filesystemdrivercoverage) · [`FilesystemDriverDerivation`](#filesystemdriverderivation) · [`FilesystemDriverProfile`](#filesystemdriverprofile) · [`FilesystemDriverReadinessLayer`](#filesystemdriverreadinesslayer) · [`FilesystemDriverReadinessReport`](#filesystemdriverreadinessreport) · [`FilesystemDriverTarget`](#filesystemdrivertarget) · [`FilesystemMetadataPatch`](#filesystemmetadatapatch) · [`FilesystemMutationModel`](#filesystemmutationmodel) · [`FilesystemNodeId`](#filesystemnodeid) · [`FilesystemNodeInfo`](#filesystemnodeinfo) · [`FilesystemNodeKind`](#filesystemnodekind) · [`FilesystemOpenOptions`](#filesystemopenoptions) · [`FilesystemOptimization`](#filesystemoptimization) · [`FilesystemOptimizationAdapters`](#filesystemoptimizationadapters) · [`FilesystemOptimizationAdapters.HardLinkDeduplicator`](#filesystemoptimizationadaptershardlinkdeduplicator) · [`FilesystemOptimizationAdapters.SymbolicLinkDeduplicator`](#filesystemoptimizationadapterssymboliclinkdeduplicator) · [`FilesystemOptimizationExtensions`](#filesystemoptimizationextensions) · [`FilesystemOptimizationFeatures`](#filesystemoptimizationfeatures) · [`FilesystemOptimizationOptions`](#filesystemoptimizationoptions) · [`FilesystemSchemaPresets`](#filesystemschemapresets) · [`FilesystemSnapshotDirectoryEntry`](#filesystemsnapshotdirectoryentry) · [`FilesystemSnapshotNode`](#filesystemsnapshotnode) · [`FormatCapabilities`](#formatcapabilities) · [`FormatCategory`](#formatcategory) · [`FormatCreateOptions`](#formatcreateoptions) · [`FormatDetectionSignature`](#formatdetectionsignature) · [`FormatHeaderMatch`](#formatheadermatch) · [`FormatHealth`](#formathealth) · [`FormatHelpers`](#formathelpers) · [`FormatMethodInfo`](#formatmethodinfo) · [`FormatOptionDescriptor`](#formatoptiondescriptor) · [`FormatOptionKind`](#formatoptionkind) · [`FormatRegistry`](#formatregistry) · [`HardLinkDeduplicationSemantics`](#hardlinkdeduplicationsemantics) · [`IArchiveCreatable`](#iarchivecreatable) · [`IArchiveDefragmentable`](#iarchivedefragmentable) · [`IArchiveFormatOperations`](#iarchiveformatoperations) · [`IArchiveInMemoryExtract`](#iarchiveinmemoryextract) · [`IArchiveLayoutMap`](#iarchivelayoutmap) · [`IArchiveModifiable`](#iarchivemodifiable) · [`IArchivePurgeable`](#iarchivepurgeable) · [`IArchiveShrinkable`](#iarchiveshrinkable) · [`IArchiveWriteConstraints`](#iarchivewriteconstraints) · [`IAsyncArchiveOperations`](#iasyncarchiveoperations) · [`IAudioContainerFormat`](#iaudiocontainerformat) · [`IAudioDemuxSource`](#iaudiodemuxsource) · [`IAudioMuxTarget`](#iaudiomuxtarget) · [`IAudioPcmSource`](#iaudiopcmsource) · [`IAudioPcmTarget`](#iaudiopcmtarget) · [`IBlockDeviceFilesystemDriverProvider`](#iblockdevicefilesystemdriverprovider) · [`IBlockDeviceProvider`](#iblockdeviceprovider) · [`IBuildingBlock`](#ibuildingblock) · [`IContainerRemuxable`](#icontainerremuxable) · [`IFileInternalChunkMover`](#ifileinternalchunkmover) · [`IFileInternalLayoutMap`](#ifileinternallayoutmap) · [`IFilesystemBlockMover`](#ifilesystemblockmover) · [`IFilesystemDriverAdapter`](#ifilesystemdriveradapter) · [`IFilesystemDriverProvider`](#ifilesystemdriverprovider) · [`IFilesystemDriverReadinessProvider`](#ifilesystemdriverreadinessprovider) · [`IFilesystemExtentMap`](#ifilesystemextentmap) · [`IFilesystemFileHandle`](#ifilesystemfilehandle) · [`IFilesystemMetadataMover`](#ifilesystemmetadatamover) · [`IFilesystemPlaceable`](#ifilesystemplaceable) · [`IFilesystemScrambleable`](#ifilesystemscrambleable) · [`IFilesystemSession`](#ifilesystemsession) · [`IFilesystemTransaction`](#ifilesystemtransaction) · [`IFormatDescriptor`](#iformatdescriptor) · [`IFormatDetectionSource`](#iformatdetectionsource) · [`IFormatOptionsSchema`](#iformatoptionsschema) · [`IFormatValidator`](#iformatvalidator) · [`ILayoutOptimizable`](#ilayoutoptimizable) · [`IPartitionEditable`](#ipartitioneditable) · [`IRandomAccessBlockDevice`](#irandomaccessblockdevice) · [`IRandomAccessBlockDeviceProvider`](#irandomaccessblockdeviceprovider) · [`IRawTrackDevice`](#irawtrackdevice) · [`IRawTrackDeviceProvider`](#irawtrackdeviceprovider) · [`IStreamFormatOperations`](#istreamformatoperations) · [`ISymbolicLinkDeduplicationLayout`](#isymboliclinkdeduplicationlayout) · [`ISyntheticEntryNames`](#isyntheticentrynames) · [`IWipeEmpty`](#iwipeempty) · [`InnerFsDetector`](#innerfsdetector) · [`IssueSeverity`](#issueseverity) · [`LayoutAnalysis`](#layoutanalysis) · [`LayoutPatch`](#layoutpatch) · [`LayoutProfile`](#layoutprofile) · [`LayoutRebuildOptions`](#layoutrebuildoptions) · [`LayoutReclaim`](#layoutreclaim) · [`MagicSignature`](#magicsignature) · [`MediaProfile`](#mediaprofile) · [`MediaProfileLookup`](#mediaprofilelookup) · [`MetadataPlacementProfile`](#metadataplacementprofile) · [`MetadataPlacementRule`](#metadataplacementrule) · [`MetadataZone`](#metadatazone) · [`MethodNameParser`](#methodnameparser) · [`ModifyRebuilder`](#modifyrebuilder) · [`MutableRebuildFilesystemSession`](#mutablerebuildfilesystemsession) · [`MutableRebuildFilesystemSession.RebuildImage`](#mutablerebuildfilesystemsessionrebuildimage) · [`MutableRebuildFilesystemSession.ValidateImage`](#mutablerebuildfilesystemsessionvalidateimage) · [`PartitionBlockDevice`](#partitionblockdevice) · [`PlacementOptions`](#placementoptions) · [`PlacementZone`](#placementzone) · [`RawDiskShrinkRebuilder`](#rawdiskshrinkrebuilder) · [`RawTrackInfo`](#rawtrackinfo) · [`ReadOnlyFilesystemSnapshotSession`](#readonlyfilesystemsnapshotsession) · [`RebuildFilesystemEntry`](#rebuildfilesystementry) · [`RebuildVerb`](#rebuildverb) · [`ScrambleOptions`](#scrambleoptions) · [`SpoolingReadOnlyFileHandle`](#spoolingreadonlyfilehandle) · [`StreamBlockDevice`](#streamblockdevice) · [`SymlinkResolver`](#symlinkresolver) · [`UnusedSpaceWiper`](#unusedspacewiper) · [`ValidationIssue`](#validationissue) · [`ValidationLevel`](#validationlevel) · [`ValidationResult`](#validationresult) · [`WholeImageRebuildCommitter`](#wholeimagerebuildcommitter) #### `AlgorithmFamily` @@ -6383,6 +6383,15 @@ Implements `IEquatable`. | `InMemory` | `static ArchiveInputInfo InMemory(string archiveName, byte[] content)` | Creates an in-memory input whose content comes from `content` rather than a file on disk. | | `ReadContent` | `byte[] ReadContent()` | Returns the input's bytes: the in-memory content when present, otherwise the file at `FullPath`. Descriptors should call this instead of `File.ReadAllBytes(FullPath)` so they transparently support in-memory (temp-free) creation and conversion. | +#### `ArchiveMutationOptions` + +Per-operation options for mutating an existing archive or filesystem image. + +| Member | Signature | Summary | +| --- | --- | --- | +| `ArchiveMutationOptions` | `ArchiveMutationOptions()` | | +| `Password` | `string Password { get; init; }` | Password or passphrase required to unlock the existing container. | + #### `ArchiveShrinker` Generic shrink driver for any archive whose descriptor implements `IArchiveCreatable`. Lists the archive's entries, extracts them to memory, and re-creates a fresh archive into the output stream using the smallest canonical size that fits (from `CanonicalSizes`). Works as the default `Shrink` implementation for most R/W filesystems: the "rebuild via WORM" pattern implicitly defragments as a side effect. Format-specific shrinkers can override for efficiency. @@ -7325,7 +7334,9 @@ Implements `IArchivePurgeable`. | Member | Signature | Summary | | --- | --- | --- | | `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Adds files to an existing instance, replacing entries with the same logical path/name. Default implementation: descriptors that also implement `IArchiveFormatOperations` and `IArchiveCreatable` get a verified extract → edit → re-create implementation through `EditViaRebuild`. Formats with a cheaper native editor override it. | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs, ArchiveMutationOptions options)` | Adds or replaces files while supplying operation-scoped credentials. | | `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes the named entries from an existing instance. Passing every entry name yields an empty container/image where the format permits one. Default implementation: a verified extract → drop-named-files → re-create edit through `EditViaRebuild`. Native implementations may instead unlink/free in place and optionally wipe released storage. | +| `Remove` | `void Remove(Stream archive, string[] entryNames, ArchiveMutationOptions options)` | Removes entries while supplying operation-scoped credentials. | #### `IArchivePurgeable` @@ -8068,6 +8079,7 @@ Generic, round-trip-verified extract → re-create engine shared by maintenance | Member | Signature | Summary | | --- | --- | --- | +| `CreateScratchStream` | `static FileStream CreateScratchStream()` | A writable scratch stream not bounded by byte[] / MemoryStream size. | | `EditViaRebuild` | `static void EditViaRebuild(Stream archive, IArchiveFormatOperations ops, IArchiveCreatable creator, Action mutate)` | Rebuild-based edit used by the generic modifier. Mutation and validation happen off to the side; the original is overwritten only after a valid staged result exists. | | `PurgeViaModifier` | `static void PurgeViaModifier(Stream archive, IArchiveFormatOperations ops, IArchiveModifiable modifier)` | Transactional purge for a mutable container. The modifier operates on a staged copy and the caller's stream is replaced only after the result lists successfully with every original live entry gone. | | `RebuildInPlace` | `static void RebuildInPlace(Stream archive, IArchiveFormatOperations ops, IArchiveCreatable creator, IReadOnlyDictionary formatSpecific = null, Action onProgress = null, CancellationToken cancellationToken = null)` | Rebuilds into a scratch file, verifies it, and only then replaces the caller-supplied stream. Cancellation is honoured until commit starts; once commit begins it runs to completion so a cancellation cannot leave the original half-overwritten. | diff --git a/Hawkynt.FileFormats.Archives/REFERENCE.md b/Hawkynt.FileFormats.Archives/REFERENCE.md index 27326b100..f7abd6dec 100644 --- a/Hawkynt.FileFormats.Archives/REFERENCE.md +++ b/Hawkynt.FileFormats.Archives/REFERENCE.md @@ -9840,7 +9840,7 @@ Writes a Long Range Zip (lrzip) container with the LZMA subtype. Other methods a Commodore 64 Lynx/LNX archive. The format stores a textual PETSCII-ish directory and uncompressed file extents in 254-byte blocks mirroring a 1541 sector with its two link bytes removed. -Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`, `IFormatOptionsSchema`, `IWipeEmpty`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `IFormatOptionsSchema`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -9862,12 +9862,15 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati | `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Performs the create operation. | | `Defragment` | `void Defragment(Stream archive)` | Lynx data extents are inherently contiguous and ordered by the directory. Defragmentation therefore consists of validating that layout and dropping transport/trailing padding after the last allocated archive block; intrinsic per-block padding is part of the format. | | `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | Performs the defragment operation. | -| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the layout. | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the exact byte layout: live directory text, directory padding, REL side sectors, logical file bytes, per-file block padding, and any trailer beyond the archive allocation. Explicit free extents make generic forensic wiping safe for this format. | | `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | Performs the extract entry to memory operation. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | | `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | | `OpenEntry` | `Stream OpenEntry(Stream archive, string entryName, string password)` | Performs the open entry operation. | +| `Purge` | `void Purge(Stream archive)` | Removes all live entries in one pass while preserving the input archive's BASIC preamble and Lynx signature. The empty directory is emitted at its minimum one-block allocation. | | `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes entries by closing their allocated block range and truncating the shifted tail. REL side-sector blocks are removed together with their data blocks. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Rebuilds the directory at its smallest whole-254-byte allocation and copies the complete existing data area byte-for-byte. This preserves REL side sectors and file-type metadata, while reclaiming directory blocks left behind after removals and any trailing transport data. | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Zeros only byte ranges that the Lynx layout proves unused: directory padding, payload block padding and trailing transport data. Live file bytes and REL side-sector metadata are retained. | ### Namespace `FileFormat.Lz4` @@ -20814,11 +20817,11 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati #### `ZipLayoutMap` -Walks the ZIP central directory and emits the byte-level layout of every local file header, compressed data payload, the central directory itself, and the EOCD record as `DefragBlockInfo` tiles. +Walks a ZIP central directory and emits a fail-closed byte-level layout of local headers, compressed payloads, optional data descriptors, the central directory and EOCD. | Member | Signature | Summary | | --- | --- | --- | -| `Enumerate` | `static IEnumerable Enumerate(Stream archive)` | Enumerates the value. | +| `Enumerate` | `static IEnumerable Enumerate(Stream archive)` | | #### `ZipModifier` diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index 39bbe9968..954c10597 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -280,7 +280,7 @@ ops.Remove(image, ["var/log/old.log"]); -Every public and protected member of all 833 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.FileSystems/REFERENCE.md). +Every public and protected member of all 836 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.FileSystems/REFERENCE.md). @@ -346,4 +346,4 @@ If this project saves you time or money, consider supporting its development: ## 📜 License -Licensed under LGPL-3.0-or-later — see the repository [LICENSE](https://github.com/Hawkynt/CompressionWorkbench/blob/main/LICENSE). \ No newline at end of file +Licensed under LGPL-3.0-or-later — see the repository [LICENSE](https://github.com/Hawkynt/CompressionWorkbench/blob/main/LICENSE). diff --git a/Hawkynt.FileFormats.FileSystems/REFERENCE.md b/Hawkynt.FileFormats.FileSystems/REFERENCE.md index 99dd39845..9f995e2c2 100644 --- a/Hawkynt.FileFormats.FileSystems/REFERENCE.md +++ b/Hawkynt.FileFormats.FileSystems/REFERENCE.md @@ -196,48 +196,56 @@ Implements `IDisposable`. #### `CsoFormatDescriptor` -PSP CSO / ZSO compressed ISO image. Layout after the 4-byte magic (`CISO` for CSO, `ZISO` for LZ4-compressed ZSO): uint32 header_size, uint64 uncompressed_size, uint32 block_size, uint8 version, uint8 align, uint16 reserved, then an index table of `N = uncompressed_size / block_size + 1` uint32 entries (high bit = stored/uncompressed, low 31 bits = file offset). This descriptor surfaces each compressed block as a raw blob — it does NOT decompress the blocks (consumers can further process with zlib for CSO or LZ4 for ZSO). References: `https://github.com/unknownbrackets/maxcso` — maxcso — maintained CSO/ZSO tool; its docs describe the CSO v1/v2 and ZSO layoutsThe format originates in PSP homebrew (ciso); there is no official Sony documentation +PSP CSO v1/v2 and ZSO compressed ISO image. -Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `ILayoutOptimizable`, `ISyntheticEntryNames`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | | `CsoFormatDescriptor` | `CsoFormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Replaces blocks named `blocks/block_NNNNN.bin` (5-digit zero-padded index) with the supplied payloads. Each input must be exactly the container's block_size bytes. Other input names are ignored. | -| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Emits a fresh CSO v1 stream. Inputs are concatenated in supplied order to form the uncompressed payload (the caller is responsible for ensuring the result is a valid PSP ISO if PSP semantics matter). | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | -| `Remove` | `void Remove(Stream archive, string[] entryNames)` | "Removes" blocks by writing block_size zero bytes through `WriteBlock`, which compresses the zero slab to its minimum DEFLATE encoding and zero-pads the on-disk slack. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `SyntheticEntryNames` | `IReadOnlySet SyntheticEntryNames { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Replaces logical block_NNNNN.bin entries transactionally. | +| `AnalyzeLayout` | `LayoutAnalysis AnalyzeLayout(Stream image)` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Defragment` | `void Defragment(Stream archive)` | Canonical repack: ordered blocks, align=0, no stale/orphaned body bytes. | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Describes the real container byte layout; only bytes after the final index are free. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Purge` | `void Purge(Stream archive)` | Leaves a valid empty container of the same CSO/ZSO variant and block geometry. | +| `RebuildStreaming` | `void RebuildStreaming(Stream source, Stream target, LayoutRebuildOptions options)` | Reblocks and recompresses while preserving the source CSO/ZSO variant and logical ISO bytes. | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | Clears logical blocks to zero; it does not edit the ISO 9660 directory tree inside them. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Rebuilds canonically and uses it only when it is smaller; otherwise copies the source through. | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Wipes dead bytes by canonicalizing into the beginning of the same-sized stream and zeroing the now-unindexed tail. If the local codecs cannot reproduce the image no larger than the source, no byte is touched. | #### `CsoInPlaceModifier` -In-place block-level mutator for PSP CSO v1 images. Lets callers replace individual decompressed blocks without rewriting the whole container. Semantics for `WriteBlock`: If the new compressed payload fits inside the old block's on-disk slot, the payload is written at the same offset and trailing slack is zero-padded — the index table and every other block's bytes are unchanged.Otherwise the payload is appended at the current end of stream and the block's index entry is updated to point at the new location. The old in-place bytes become orphaned (defrag-recoverable).CSO v2 / ZSO (LZ4) are out of scope; only the v1 header layout with align=0 is supported. The modifier refuses to operate on streams whose header reports a different version or a non-zero align (because the offset-shift semantics would silently misplace the new block). +Block-level mutator for CSO v1/v2 and ZSO images. | Member | Signature | Summary | | --- | --- | --- | -| `WriteBlock` | `static void WriteBlock(Stream image, int blockIndex, ReadOnlySpan newUncompressedData)` | Replaces block `blockIndex`'s content with `newUncompressedData` (which must be exactly `block_size` bytes long, matching the container's geometry). The payload is DEFLATE-compressed; if the result is smaller than the slab it's written compressed, otherwise stored uncompressed (with the index entry's high bit set). | +| `WriteBlock` | `static void WriteBlock(Stream image, int blockIndex, ReadOnlySpan newUncompressedData)` | Replaces one logical block. `newUncompressedData` must be exactly block_size bytes; for a partial final block only its logical prefix is retained, matching CSO semantics. | #### `CsoWriter` -Writes a PSP CSO v1 ("CISO") compressed-ISO container from scratch (WORM). Layout (24-byte header + (N+1)·uint32 index + N compressed blocks): $00..$03: magic "CISO"$04..$07: uint32 LE header_size = 24$08..$0F: uint64 LE uncompressed_size$10..$13: uint32 LE block_size (this writer uses 2048 = ISO 9660 sector)$14: uint8 version = 1$15: uint8 align (left-shift applied to index offsets; this writer uses 0)$16..$17: uint16 reserved = 0$18..(header+4·(N+1)): index table. bit 31 set = stored uncompressed.Each block: raw DEFLATE bytes (no zlib header) of one block_size-aligned slab, OR the slab verbatim when the compressed output is not smaller than the slab.ZSO (LZ4) and CSO v2 are out of scope; see `CsoFormatDescriptor`. +Writes PSP CSO v1/v2 and ZSO compressed-ISO containers. | Member | Signature | Summary | | --- | --- | --- | | `CsoWriter` | `CsoWriter()` | | -| `DefaultBlockSize` | `const int DefaultBlockSize` | Default block size — 2048 = one ISO 9660 cooked sector. | -| `Build` | `static byte[] Build(ReadOnlySpan uncompressedData, int blockSize = 2048)` | Builds a CSO v1 stream that, when fully decompressed, yields `uncompressedData`. Blocks of `blockSize` bytes are each DEFLATE-compressed; if the compressed output is not smaller than the original slab, the slab is stored verbatim and its index entry gets the `IndexUncompressedFlag` bit set. | +| `DefaultBlockSize` | `const int DefaultBlockSize` | Default block size — one cooked ISO 9660 sector. | +| `Build` | `static byte[] Build(ReadOnlySpan uncompressedData, int blockSize = 2048)` | Builds a canonical CSO v1 stream around `uncompressedData`. | ### Namespace `FileFormat.Dmg` @@ -535,7 +543,7 @@ Writers for the two firmware text formats, and the shared step that turns a crea | `MetadataName` | `const string MetadataName` | The name the reader gives the rendered summary. | | `PayloadName` | `const string PayloadName` | The name the reader gives the flat payload. | | `ImageFrom` | `static FirmwareImage ImageFrom(IReadOnlyList inputs, string sourceFormat)` | The image a create or edit describes: the single payload input as the bytes, and the addresses read out of `metadata.ini` when the caller passes the one the reader rendered. An input list with no payload describes an image with no data, which both formats can write. | -| `WriteIntelHex` | `static void WriteIntelHex(Stream output, FirmwareImage image, int bytesPerRecord = 16)` | Writes `image` as Intel HEX: type-04 extended-linear-address records whenever the high half of the address changes, type-00 data records of at most `bytesPerRecord` bytes that never straddle a 64 KiB boundary, an optional type-05 start-linear-address record, and the type-01 end-of-file record every reader requires. | +| `WriteIntelHex` | `static void WriteIntelHex(Stream output, FirmwareImage image, int bytesPerRecord = 16)` | Writes `image` as Intel HEX: type-04 extended-linear-address records whenever the high half of the address changes, type-00 data records of at most `bytesPerRecord` bytes that never straddle a 64 KiB boundary, an optional type-03 or type-05 start-address record, and the type-01 end-of-file record every reader requires. | | `WriteTiTxt` | `static void WriteTiTxt(Stream output, FirmwareImage image, int bytesPerLine = 16)` | Writes `image` as TI-TXT: an `@AAAA` address line per segment, space-separated hex bytes at `bytesPerLine` a line, and the single `q` the format ends with. | #### `FirmwareImage` @@ -553,14 +561,15 @@ Implements `IEquatable`. | `Segments` | `IReadOnlyList> Segments { get; init; }` | | | `SourceFormat` | `string SourceFormat { get; init; }` | | | `StartAddress` | `uint? StartAddress { get; init; }` | | +| `StartSegmentAddress` | `ValueTuple? StartSegmentAddress { get; init; }` | Gets the original segmented x86 start address when an Intel HEX type-03 record supplied one. Keeping CS:IP separately matters because many CS:IP pairs map to the same linear address and therefore cannot be reconstructed from `StartAddress` alone. | | `TotalDataBytes` | `int TotalDataBytes { get; init; }` | | | `ToFlatBinary` | `byte[] ToFlatBinary(byte fill = 255)` | Flattens all segments into a single contiguous binary spanning from the lowest address to the end of the highest segment. Gaps are filled with `fill` (default `0xFF` to match flash erase state). | #### `IntelHexFormatDescriptor` -Pseudo-archive descriptor for Intel HEX firmware files. Decodes the ASCII records into a flat binary (`firmware.bin`) and surfaces a `metadata.ini` with record count, declared start address, and gap count. References: Intel "Hexadecimal Object File Format Specification", Rev. A (1988) — the defining document`https://en.wikipedia.org/wiki/Intel_HEX` — record types and checksum rules +Pseudo-archive descriptor for Intel HEX firmware files. Decodes the ASCII records into a flat binary (`firmware.bin`) and surfaces a `metadata.ini` with record count, declared start address, and sparse segment layout. References: Intel "Hexadecimal Object File Format Specification", Rev. A (1988) — the defining document`https://developerhelp.microchip.com/xwiki/bin/view/software-tools/ipe/sqtp-file-format-specification/intel-hex/` — record layout and checksum rules -Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`. | Member | Signature | Summary | | --- | --- | --- | @@ -577,9 +586,10 @@ Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IFormatDescriptor`. | `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | | `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Writes a fresh Intel HEX file: the single payload input becomes the data records, and a `metadata.ini` alongside it -- the one this descriptor's own reader renders -- supplies the base and start addresses that a flat binary cannot carry. | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Writes a fresh Intel HEX file: the single payload input becomes the data records, and a `metadata.ini` alongside it -- the one this descriptor's own reader renders -- supplies the sparse segment map and start-address form that a flat binary cannot carry. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | | `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Purge` | `void Purge(Stream archive)` | Removes all programmed bytes and start-address state while leaving the canonical valid empty Intel HEX document (the EOF record). This overrides the generic pseudo-archive purge because `firmware.bin` is a rendered view that also exists as a zero-length view of an empty image. | #### `IntelHexReader` @@ -601,37 +611,45 @@ Reader for Motorola S-Record files (`Stnn[aaaa|aaaaaa|aaaaaaaa]dd…cc`). Recogn #### `TiTxtFormatDescriptor` -Pseudo-archive descriptor for the TI-TXT firmware text format used by MSP430. Address lines (`@HHHH`) introduce contiguous byte runs; a single `q` terminates the file. Extension is intentionally empty — `.txt` is far too ambiguous — so detection relies on the first non-whitespace byte being `@`. References: Texas Instruments MSP430 programming/bootloader guides — define the TI-TXT format (@addr / data / q)`https://srecord.sourceforge.net` — SRecord tool suite — documents and converts TI-TXT (srec_ti_txt) +Pseudo-archive descriptor for the TI-TXT firmware text format used by MSP430. Address lines (`@HHHH`) introduce sparse byte runs; a line containing `q` terminates the file. The archive view exposes the logical firmware as `firmware.bin` plus a rendered `metadata.ini` summary. References: `https://downloads.ti.com/docs/esd/SPRU513P/Content/SPRU513P_HTML/hex-conversion-utility-description.html` — TI Hex Conversion Utility, TI-TXT format`https://www.ti.com/lit/pdf/slau131` — Texas Instruments MSP430 programming documentation`https://srecord.sourceforge.net/man/man5/srec_ti_txt.5.html` — SRecord TI-TXT documentation; used only as an interoperability reference -Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveWriteConstraints`, `IFormatDescriptor`, `ISyntheticEntryNames`. | Member | Signature | Summary | | --- | --- | --- | | `TiTxtFormatDescriptor` | `TiTxtFormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Writes a fresh TI-TXT file: the single payload input becomes the data lines under an `@address` taken from a `metadata.ini` alongside it, and the file ends with the `q` the format requires. | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `AcceptedInputsDescription` | `string AcceptedInputsDescription { get; }` | | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `MaxTotalArchiveSize` | `long? MaxTotalArchiveSize { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `SyntheticEntryNames` | `IReadOnlySet SyntheticEntryNames { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Replaces the single logical firmware payload and/or moves its base address. A metadata-only base-address edit shifts every sparse section by the same delta so address holes remain holes rather than becoming explicit 0xFF data. | +| `CanAccept` | `bool CanAccept(ArchiveInputInfo input, out string reason)` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Defragment` | `void Defragment(Stream archive)` | Canonicalises section ordering and record wrapping without flattening sparse holes. The reader merges only exactly adjacent sections; the writer then emits one address record per contiguous run and 16 bytes per full data line. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Purge` | `void Purge(Stream archive)` | Erases all live firmware bytes while leaving a valid empty TI-TXT document. | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removing the logical firmware payload leaves the valid empty TI-TXT form, `q`. | #### `TiTxtReader` -Reader for TI-TXT MSP430 text firmware files. Addresses are introduced by `@HHHH` lines; data lines follow as space-separated hex bytes (typically 16 per line); a single `q` token terminates the file. +Parser for Texas Instruments TI-TXT firmware images. Address records introduce sparse byte runs and a line containing only `q` terminates the document. | Member | Signature | Summary | | --- | --- | --- | | `TiTxtReader` | `TiTxtReader()` | | -| `Read` | `static FirmwareImage Read(string text)` | Parses a TI-TXT document into a `FirmwareImage`. | +| `Read` | `static FirmwareImage Read(string text)` | Parses a TI-TXT document into its sparse firmware image. | ### Namespace `FileFormat.Ipsw` @@ -639,31 +657,36 @@ Reader for TI-TXT MSP430 text firmware files. Addresses are introduced by `@HHHH #### `IpswFormatDescriptor` -Apple IPSW / OTA firmware package. An IPSW is just a ZIP file (with an Apple-specific layout). Rather than surfacing entries as a flat generic ZIP, this descriptor lifts the well-known Apple artifacts (`BuildManifest.plist`, `Firmware/` subtree, `LLB.*`, `iBSS.*`, `iBEC.*`, `iBoot.*`, root-filesystem `*.dmg`) into first-class canonical entries. Everything else is exposed under `other/`. This is a compound-extension descriptor (`.ipsw`, `.otazip`): magic is empty so it does not steal generic ZIPs. Read-only; the plist and DMG payloads are emitted as raw bytes — no plist parsing or DMG mounting. References: `https://theapplewiki.com` — The Apple Wiki (formerly The iPhone Wiki) — community IPSW documentation`https://github.com/blacktop/ipsw` — ipsw — maintained IPSW research and extraction tool`https://en.wikipedia.org/wiki/IPSW` — Wikipedia +Apple IPSW / OTA firmware package. IPSW files are ZIP containers whose entry paths are semantically significant: manifests refer to firmware members by their ZIP path, so the descriptor exposes those paths losslessly instead of projecting them into a flattened view. -Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`, `IWipeEmpty`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `ISyntheticEntryNames`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | | `IpswFormatDescriptor` | `IpswFormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Adds (or replaces by ZIP path) entries inside an existing IPSW. Routes through `IpswInPlaceModifier` — only the central directory, EOCD, and the appended LFH + payload are touched. Synthetic canonical entries are silently dropped. | -| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Emits a fresh IPSW (ZIP) container from the supplied inputs. Synthetic canonical entries the descriptor surfaces on read (`FULL.ipsw`, `metadata.ini`) are silently dropped — they aren't real ZIP entries. All other inputs are stored under their `ArchiveName`. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `SyntheticEntryNames` | `IReadOnlySet SyntheticEntryNames { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Defragment` | `void Defragment(Stream archive)` | | +| `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | | | `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | -| `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes named ZIP entries from an existing IPSW. Routes through `IpswInPlaceModifier` — the LFH + compressed payload of the dropped entry are zero-wiped and the central directory is rewritten. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `OpenEntry` | `Stream OpenEntry(Stream archive, string entryName, string password)` | | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | | +| `Shrink` | `void Shrink(Stream input, Stream output)` | | #### `IpswInPlaceModifier` @@ -682,7 +705,7 @@ In-place modifier for Apple IPSW packages. An IPSW is just a ZIP file with an Ap Commodore 64 Lynx/LNX archive. The format stores a textual PETSCII-ish directory and uncompressed file extents in 254-byte blocks mirroring a 1541 sector with its two link bytes removed. -Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`, `IFormatOptionsSchema`, `IWipeEmpty`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `IFormatOptionsSchema`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -704,12 +727,15 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati | `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Performs the create operation. | | `Defragment` | `void Defragment(Stream archive)` | Lynx data extents are inherently contiguous and ordered by the directory. Defragmentation therefore consists of validating that layout and dropping transport/trailing padding after the last allocated archive block; intrinsic per-block padding is part of the format. | | `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | Performs the defragment operation. | -| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the layout. | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the exact byte layout: live directory text, directory padding, REL side sectors, logical file bytes, per-file block padding, and any trailer beyond the archive allocation. Explicit free extents make generic forensic wiping safe for this format. | | `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | Performs the extract entry to memory operation. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | | `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | | `OpenEntry` | `Stream OpenEntry(Stream archive, string entryName, string password)` | Performs the open entry operation. | +| `Purge` | `void Purge(Stream archive)` | Removes all live entries in one pass while preserving the input archive's BASIC preamble and Lynx signature. The empty directory is emitted at its minimum one-block allocation. | | `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes entries by closing their allocated block range and truncating the shifted tail. REL side-sector blocks are removed together with their data blocks. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Rebuilds the directory at its smallest whole-254-byte allocation and copies the complete existing data area byte-for-byte. This preserves REL side sectors and file-type metadata, while reclaiming directory blocks left behind after removals and any trailing transport data. | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Zeros only byte ranges that the Lynx layout proves unused: directory padding, payload block padding and trailing transport data. Live file bytes and REL side-sector metadata are retained. | ### Namespace `FileFormat.Mdf` @@ -1373,34 +1399,39 @@ Implements `IEquatable
`. #### `UefiFvFormatDescriptor` -UEFI PI Firmware Volume (`.fv`/`.fd`) archive surface. FFS files are exposed as `{GUID}_{TYPE_TAG}.bin`; standalone volumes can be created and ordinary FFS2 records can be added/replaced/removed through erased free space. References: `https://uefi.org/specifications` — UEFI Platform Initialization (PI) Specification, Volume 3: Firmware Storage Design`https://github.com/LongSoft/UEFITool` — UEFITool firmware-volume parser/editor +UEFI PI Firmware Volume archive surface. Standard FFS2/FFS3 files are exposed as `{GUID}_{TYPE_TAG}.bin`; mutable unsigned volumes support transactional edits, erase-aware wiping, purge, and offline consolidation. References: UEFI PI Specification 1.10 Volume III (Firmware Storage) and EDK II as an interoperability oracle. The implementation is independently written. -Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | | `UefiFvFormatDescriptor` | `UefiFvFormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | | `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | | | `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Defragment` | `void Defragment(Stream archive)` | | +| `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Purge` | `void Purge(Stream archive)` | | | `Remove` | `void Remove(Stream archive, string[] entryNames)` | | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Restores unused bytes to the FV's declared erase value. For erase-polarity-one flash this is 0xFF rather than zero; writing zero would turn free space into apparently programmed data and make the firmware volume structurally invalid. | #### `UefiFvReader` -Reader for UEFI Platform Initialization (PI) Firmware Volumes. Locates the FV header by scanning for the `_FVH` signature at offset 40 from the start of each 16-byte-aligned candidate (UEFI PI Volume 3). Walks the FFS file list and returns one `FfsFile` record per live file. +Reader for UEFI PI firmware volumes and standard FFS2/FFS3 file records. | Member | Signature | Summary | | --- | --- | --- | @@ -1447,6 +1478,8 @@ Implements `IEquatable`. | `Attributes` | `uint Attributes { get; init; }` | | | `BlockMap` | `IReadOnlyList> BlockMap { get; init; }` | | | `Checksum` | `ushort Checksum { get; init; }` | | +| `EraseByte` | `byte EraseByte { get; }` | | +| `ErasePolarity` | `bool ErasePolarity { get; }` | | | `ExtHeaderOffset` | `ushort ExtHeaderOffset { get; init; }` | | | `FileSystemGuid` | `Guid FileSystemGuid { get; init; }` | | | `FvLength` | `ulong FvLength { get; init; }` | | @@ -5166,9 +5199,9 @@ Represents an ecryptfs entry. #### `EcryptfsFormatDescriptor` -Read-only descriptor for eCryptfs per-file encryption containers. eCryptfs (Linux) stacks on top of any underlying FS and stores each encrypted file with a 4-byte big-endian marker `0x3C81B7F5` at offset 0 followed by an 8-byte decrypted size, 4-byte flags, and 4-byte extent-size hint. Decryption requires the user's passphrase + EFEK packets — out of scope. The encrypted payload is surfaced as a single opaque entry along with the parsed header metadata. References: `https://docs.kernel.org/filesystems/ecryptfs.html` — Linux kernel eCryptfs documentation`https://github.com/torvalds/linux/tree/master/fs/ecryptfs` — mainline implementation (`ecryptfs_kernel.h` defines the file-header marker + packet layout)`https://en.wikipedia.org/wiki/ECryptfs` — Wikipedia overview +Descriptor for one eCryptfs lower file. eCryptfs is a stacked filesystem: every lower file carries its own encrypted upper-file payload, size, extent geometry and authentication-token packet set. The implementation supports the ordinary passphrase path using the Linux eCryptfs AES-128/192/256 packet conventions. Private-key authentication and xattr-only metadata are deliberately rejected because their required key/xattr material is not contained in a standalone stream. -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -5182,30 +5215,40 @@ Implements `IArchiveFormatOperations`, `IFormatDescriptor`. | `Extensions` | `IReadOnlyList Extensions { get; }` | | | `Family` | `AlgorithmFamily Family { get; }` | | | `Id` | `string Id { get; }` | | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | eCryptfs has no fixed byte magic: its marker is two random 32-bit words whose XOR equals 0x3C81B7F5. Static signature matching cannot express that relation, so detection falls back to the extension and the reader performs the authoritative relational check. | | `Methods` | `IReadOnlyList Methods { get; }` | | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | | `List` | `List List(Stream stream, string password)` | | +| `Purge` | `void Purge(Stream archive)` | Removes the logical payload without requiring the FEK: set plaintext length to zero and truncate all ciphertext extents while retaining valid key metadata. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Drops any bytes after the encrypted extents implied by the header. This is the only meaningful compact/shrink operation for a single eCryptfs lower file. | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Clears metadata padding and bytes beyond the canonical encrypted extent set. Ciphertext inside the final live extent is never touched: plaintext tail zeroing would require the passphrase, which the wipe interface intentionally does not carry. | #### `EcryptfsReader` -Reads eCryptfs per-file encryption headers. eCryptfs is a stacking file-level encryption filesystem (Linux) — every encrypted file is stored on the lower filesystem as a regular file whose first page is a metadata header followed by AES-CBC ciphertext extents. The header starts with a 4-byte big-endian marker (`0x3C81B7F5`) so the individual on-disk container is well-defined and detectable. Decryption requires the user's mount passphrase + EFEK (Encrypted File Encryption Key) tag-3 / tag-11 packets and is OUT OF SCOPE; this reader surfaces the parsed header + the encrypted payload as a single opaque entry. File header layout (big-endian, file offset 0): 0x00 u32 marker == 0x3C81B7F5 0x04 u64 decrypted-size (plaintext length, host-endian on Linux) 0x0C u32 flags 0x10 u32 extent-size (typically 4096) 0x14 ... EFEK packets, tag-3 / tag-11 OpenPGP-style ~0x800 start of AES-CBC ciphertext extents +Reader for an eCryptfs lower file. eCryptfs is a stacked filesystem, so one lower file represents one encrypted upper file rather than a complete volume. Implements `IDisposable`. | Member | Signature | Summary | | --- | --- | --- | -| `EcryptfsReader` | `EcryptfsReader(Stream stream)` | Initializes a new instance of `EcryptfsReader`. | -| `EcryptfsMarker` | `const uint EcryptfsMarker` | Defines the ecryptfs marker constant value. | -| `DecryptedSize` | `ulong DecryptedSize { get; }` | Gets or sets the decrypted size. | -| `Entries` | `IReadOnlyList Entries { get; }` | Gets the entries. | -| `ExtentSize` | `uint ExtentSize { get; }` | Gets or sets the extent size. | -| `Flags` | `uint Flags { get; }` | Gets or sets the flags. | -| `Marker` | `uint Marker { get; }` | Gets or sets the marker. | -| `ValidHeader` | `bool ValidHeader { get; }` | Gets a value indicating whether valid header. | -| `Dispose` | `void Dispose()` | Releases resources held by this instance. | -| `Extract` | `byte[] Extract(EcryptfsEntry entry)` | Decodes the supplied input. | +| `EcryptfsReader` | `EcryptfsReader(Stream stream)` | | +| `CanonicalLength` | `long CanonicalLength { get; }` | Canonical lower-file length implied by the plaintext size and extent geometry. | +| `CipherDescription` | `string CipherDescription { get; }` | Human-readable cipher of the first passphrase packet, when present. | +| `DecryptedSize` | `ulong DecryptedSize { get; }` | Plaintext file length stored in the lower-file header. | +| `Entries` | `IReadOnlyList Entries { get; }` | | +| `ExtentSize` | `uint ExtentSize { get; }` | Encryption extent size in bytes. | +| `FileVersion` | `byte FileVersion { get; }` | eCryptfs lower-file format version. | +| `Flags` | `uint Flags { get; }` | Raw eCryptfs file flags, including the version in the top byte. | +| `HeaderExtentCount` | `ushort HeaderExtentCount { get; }` | Number of header extents preceding ciphertext data. | +| `MarkerWord` | `uint MarkerWord { get; }` | The random first word of the two-word marker pair stored on disk. | +| `Marker` | `uint Marker { get; }` | The validated eCryptfs marker relation constant. | +| `MetadataSize` | `int MetadataSize { get; }` | Total metadata region at the beginning of the lower file. | +| `PassphraseSignature` | `string PassphraseSignature { get; }` | Raw eight-byte authentication-token signature as lowercase hexadecimal. | +| `Dispose` | `void Dispose()` | | +| `ExtractContent` | `byte[] ExtractContent(string password)` | Decrypts the single logical upper-file payload with a passphrase. | +| `ValidatePassword` | `void ValidatePassword(string password)` | Validates a passphrase against the authentication-token packet without materializing plaintext in memory. The ciphertext is streamed through the normal decoder into `Null`, so the same key and extent path used by extraction is exercised before a destructive mutation starts. | ### Namespace `FileSystem.Efs` @@ -5556,7 +5599,7 @@ Builds exFAT filesystem images that Windows 10+ actually mounts. Default layout: ### Namespace `FileSystem.Ext` -[`ExtBlockMover`](#extblockmover) · [`ExtEntry`](#extentry) · [`ExtExtentMap`](#extextentmap) · [`ExtFilesystemDriverAdapter`](#extfilesystemdriveradapter) · [`ExtFormatDescriptor`](#extformatdescriptor) · [`ExtInPlaceShrinker`](#extinplaceshrinker) · [`ExtInPlaceShrinker.ShrinkResult`](#extinplaceshrinkershrinkresult) · [`ExtModifier`](#extmodifier) · [`ExtModifier.InPlaceUnsupportedException`](#extmodifierinplaceunsupportedexception) · [`ExtReader`](#extreader) · [`ExtRemover`](#extremover) · [`ExtShrinkHelper`](#extshrinkhelper) · [`ExtShrinkHelper.ShrinkResult`](#extshrinkhelpershrinkresult) · [`ExtWriter`](#extwriter) · [`ExtWriter.ExtVersion`](#extwriterextversion) +[`ExtBlockMover`](#extblockmover) · [`ExtEntry`](#extentry) · [`ExtExtendedAttributes`](#extextendedattributes) · [`ExtExtentMap`](#extextentmap) · [`ExtFilesystemDriverAdapter`](#extfilesystemdriveradapter) · [`ExtFormatDescriptor`](#extformatdescriptor) · [`ExtInPlaceShrinker`](#extinplaceshrinker) · [`ExtInPlaceShrinker.ShrinkResult`](#extinplaceshrinkershrinkresult) · [`ExtModifier`](#extmodifier) · [`ExtModifier.InPlaceUnsupportedException`](#extmodifierinplaceunsupportedexception) · [`ExtReader`](#extreader) · [`ExtRemover`](#extremover) · [`ExtShrinkHelper`](#extshrinkhelper) · [`ExtShrinkHelper.ShrinkResult`](#extshrinkhelpershrinkresult) · [`ExtWriter`](#extwriter) · [`ExtWriter.ExtVersion`](#extwriterextversion) #### `ExtBlockMover` @@ -5591,6 +5634,16 @@ Represents an ext entry. | `Name` | `string Name { get; init; }` | Gets or sets the name. | | `Size` | `long Size { get; init; }` | Gets or sets the size. | +#### `ExtExtendedAttributes` + +Reads ext2/3/4 extended attributes and performs conservative in-inode mutations. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Read` | `static IReadOnlyDictionary Read(Stream image, string path)` | Reads all directly stored xattrs for `path`. | +| `Remove` | `static bool Remove(Stream image, string path, string name)` | Removes one in-inode xattr, returning false when it was absent. | +| `Set` | `static void Set(Stream image, string path, string name, ReadOnlySpan value)` | Creates or replaces one in-inode xattr. External-block and EA-inode values are never rewritten by this conservative mutator. | + #### `ExtExtentMap` Walks an ext2/3/4 image and yields its actual on-disk byte layout — per-file extent runs (one `DefragBlockInfo` per contiguous block range) plus metadata regions (superblock, group descriptors, block + inode bitmaps, inode table). Used by the defragment window's block-map preview. Streaming: never loads the whole image. All reads flow through a `SectorCache` so multi-TB ext4 images (a 50 TB volume's BGD table + bitmaps are tens of MB) work without OOM. @@ -6817,12 +6870,12 @@ Clean-room GFS2 (Global File System 2) image writer producing a minimal, standal #### `GlusterFsEntry` -Represents a gluster fs entry. +Represents an entry surfaced from a single GlusterFS brick backing store. | Member | Signature | Summary | | --- | --- | --- | | `GlusterFsEntry` | `GlusterFsEntry()` | | -| `Data` | `byte[] Data { get; init; }` | Gets or sets the data. | +| `Data` | `byte[] Data { get; init; }` | Gets or sets eagerly materialized data, used by synthetic metadata entries. | | `IsDirectory` | `bool IsDirectory { get; init; }` | Gets a value indicating whether is directory. | | `Name` | `string Name { get; init; }` | Gets or sets the name. | | `Offset` | `long Offset { get; init; }` | Gets or sets the offset. | @@ -6830,9 +6883,9 @@ Represents a gluster fs entry. #### `GlusterFsFormatDescriptor` -Stage 0 detection-only descriptor for GlusterFS. Honest fallback: GlusterFS has no on-disk image format. A GlusterFS volume is a logical aggregation of one or more "bricks", and every brick is just a normal directory on a local POSIX filesystem (typically XFS or ext4). Volume files live at their normal POSIX paths inside the brick directory and carry GlusterFS state in extended attributes (`trusted.gfid`, `trusted.glusterfs.dht`, `trusted.glusterfs.volume-id`, `trusted.glusterfs.pathinfo`, etc.). There is no superblock, no brick header, no portable single-file representation that this image-based pipeline can consume. We therefore stay Stage 0 permanently. The 0xCAFE5BAB magic recognised here is a workbench-internal convention for hand-dumped brick-object probes — it is not a real on-disk GlusterFS structure and no real GlusterFS deployment will produce it. Promotion to R/O would require walking a live directory tree and reading xattrs, which is outside the image-stream contract enforced by `IArchiveFormatOperations`. References: `https://docs.gluster.org` — official GlusterFS documentation (brick/xattr architecture)`https://github.com/gluster/glusterfs` — canonical source`https://en.wikipedia.org/wiki/GlusterFS` — Wikipedia overview +Read-only single-brick view for GlusterFS backing-store images, with a conservative backing-store shrink operation where it is provably metadata-safe. GlusterFS has no independent block format: a volume is a logical collection of bricks and each brick is an export directory on an ordinary filesystem with extended-attribute support. This descriptor therefore delegates real backing images to the repository's XFS or ext2/3/4 reader and surfaces the physical contents of one brick. It does not reconstruct the distributed Gluster namespace.Detection is extension-only. XFS and ext images already belong to their native descriptors and there is no Gluster-specific superblock magic with which to distinguish a brick image automatically. The former workbench-only 0xCAFE5BAB probe convention is deliberately not recognised as GlusterFS.The native XFS/ext layers can now read Gluster xattrs and conservatively mutate the common short-form/in-inode storage cases. General archive mutation remains disabled until every xattr storage form that a maintenance operation can encounter (including ext external blocks/EA inodes and XFS leaf/btree/remote attributes) is preserved. Advertising a rebuild-based mutation before then could silently discard `trusted.gfid` or `trusted.glusterfs.*` state.Shrink is the deliberate exception. For ext-backed bricks the native in-place shrinker chooses its boundary from the filesystem allocation bitmap; every allocated xattr block therefore pins the boundary just like file data and is preserved byte-for-byte. XFS-backed bricks copy through unchanged because the current XFS shrink path may rebuild the image. This is a backing-store geometry operation only; it is not Gluster's remove-brick command.Gluster volume operations such as rebalance, fix-layout, and remove-brick are explicitly outside this single-image abstraction. They coordinate multiple bricks and belong to a live cluster/volume control plane, not an offline brick image editor. References: `https://docs.gluster.org/en/latest/Administrator-Guide/Setting-Up-Volumes/` — a volume is a logical collection of export-directory bricks`https://docs.gluster.org/en/latest/Administrator-Guide/GlusterFS-Introduction/` — backing filesystems must support extended attributes`https://docs.gluster.org/en/latest/Administrator-Guide/Managing-Volumes/` — remove-brick/rebalance/fix-layout are volume operations`https://docs.kernel.org/filesystems/ext4/attributes.html` — ext4 xattr on-disk layout`https://www.kernel.org/pub/linux/utils/fs/xfs/docs/xfs_filesystem_structure.pdf` — XFS short-form attribute layout`https://github.com/gluster/glusterfs` — canonical GlusterFS implementation, dual GPLv2/LGPLv3+ -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveFormatOperations`, `IArchiveShrinkable`, `IFormatDescriptor`. | Member | Signature | Summary | | --- | --- | --- | @@ -6846,28 +6899,30 @@ Implements `IArchiveFormatOperations`, `IFormatDescriptor`. | `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | | `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | | `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. Empty by design because GlusterFS has no separate on-disk magic. | | `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Extracts entries from the supplied single-brick physical view. | +| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied brick backing-store image. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Shrinks an ext-backed brick to its highest allocated block without rebuilding file or xattr metadata. XFS-backed bricks are copied unchanged. | #### `GlusterFsReader` -Stage 0 detection-only reader for GlusterFS — permanent honest fallback. GlusterFS itself has no on-disk image format: a brick is a normal directory on a local POSIX filesystem (XFS / ext4 / ...) and volume files are stored at their normal POSIX paths inside that directory. All GlusterFS-specific state lives in extended attributes (the `trusted.gfid`, `trusted.glusterfs.dht`, `trusted.glusterfs.volume-id`, `trusted.glusterfs.pathinfo` namespace). Consequences: There is no superblock or brick header to parse.Distribution / replication state (DHT hashing → brick mapping, AFR replicate metadata, EC dispersed metadata, rebalance bookkeeping) only exists across multiple bricks on multiple hosts, not inside any single image.An R/O promotion is fundamentally incompatible with this project's image-stream contract — recognising a GlusterFS "volume" would require walking a live POSIX directory tree and reading xattrs through the host OS, which is outside the `Stream`-based `IArchiveFormatOperations` surface. The 0xCAFE5BAB magic verified by `Parse` is a workbench-internal probe convention used to dump and round-trip hand-crafted "brick object" experiments; it is not a real on-disk GlusterFS marker and no real GlusterFS deployment produces it. The reader therefore stays a thin two-entry detector (synthetic `metadata.ini` + raw `gluster-brick.bin`) and will never grow real semantics. +Reads a single GlusterFS brick backing-store image by delegating the native on-disk filesystem to the existing XFS or ext2/3/4 reader. GlusterFS does not define a separate block format. A brick is an export directory on an ordinary filesystem that supports extended attributes. The logical volume namespace and DHT/AFR/EC state span multiple bricks, while object identity and translator metadata are stored in xattrs such as `trusted.gfid` and `trusted.glusterfs.*`. Consequently this reader intentionally exposes only the physical view of one supplied backing image. It does not claim to reconstruct a Gluster volume from one brick.When the backing namespace contains Gluster's `.glusterfs` GFID index, its parent directory identifies the brick root. Entries outside that subtree and the index itself are omitted from the normal view. If no index is present, the filesystem root is used as a conservative explicit-input fallback.Gluster xattrs are readable through `ReadExtendedAttributes`. The backing accessors also support conservative inline/short-form mutation, but this brick reader remains read-only until all xattr storage forms needed by a maintenance operation can be preserved without rebuilding metadata. Implements `IDisposable`. | Member | Signature | Summary | | --- | --- | --- | -| `GlusterFsReader` | `GlusterFsReader(Stream stream)` | Initializes a new instance of `GlusterFsReader`. | -| `BrickMagic` | `static readonly byte[] BrickMagic` | Workbench-internal probe magic (0xCA 0xFE 0x5B 0xAB, 0xCAFE5BAB big-endian) used by the detector tests. Not a real GlusterFS structure — GlusterFS has no on-disk header at all. | -| `Entries` | `IReadOnlyList Entries { get; }` | Gets the entries. | -| `MagicWord` | `uint MagicWord { get; }` | Gets or sets the magic word. | -| `TrailingWord` | `uint TrailingWord { get; }` | Gets or sets the trailing word. | -| `ValidHeader` | `bool ValidHeader { get; }` | Gets a value indicating whether valid header. | -| `Dispose` | `void Dispose()` | Releases resources held by this instance. | -| `Extract` | `byte[] Extract(GlusterFsEntry entry)` | Decodes the supplied input. | +| `GlusterFsReader` | `GlusterFsReader(Stream stream)` | Initializes a reader over one brick backing-store image. | +| `BackingFileSystem` | `string BackingFileSystem { get; }` | Gets the detected backing filesystem name (`xfs` or `ext`). | +| `BrickRoot` | `string BrickRoot { get; }` | Gets the inferred brick root inside the backing filesystem. Empty means filesystem root. | +| `Entries` | `IReadOnlyList Entries { get; }` | Gets the entries in the single-brick physical view. | +| `HasGlusterIndex` | `bool HasGlusterIndex { get; }` | Gets whether a `.glusterfs` GFID index was present and used to locate the brick root. | +| `ValidHeader` | `bool ValidHeader { get; }` | Gets a value indicating whether a supported backing filesystem was found. | +| `Dispose` | `void Dispose()` | Releases backing filesystem readers and any spool created for a non-seekable input. | +| `Extract` | `byte[] Extract(GlusterFsEntry entry)` | Extracts one surfaced entry. | +| `ReadExtendedAttributes` | `IReadOnlyDictionary ReadExtendedAttributes(GlusterFsEntry entry)` | Reads the native extended attributes belonging to a surfaced brick entry. | ### Namespace `FileSystem.Gpfs` @@ -8651,9 +8706,9 @@ Represents a lustre entry. #### `LustreFormatDescriptor` -R/O descriptor for Lustre MDT/OST images via ldiskfs (ext4-compatible) reader delegation. Surfaces the ldiskfs view of a single MDT or OST backing store — NOT the Lustre logical view (which would require combining MDT inode metadata with file data striped across multiple OSTs, out of scope without live cluster metadata). Detection is extension-routed (.lustre / .ost / .mdt) and the legacy "LUSTRE" / "LUst" object-header magic at offset 0; ext4 superblock magic is deliberately NOT registered here (it would steal detection from generic ext4 images). When opened with an ldiskfs MDT/OST image (recognised by the .ost / .mdt / .lustre extension), `LustreReader` delegates the file walk to `FileSystem.Ext.ExtReader`. References: `https://www.lustre.org/` — project home`https://wiki.lustre.org/` — Lustre wiki (architecture, ldiskfs/MDT/OST layout)`https://en.wikipedia.org/wiki/Lustre_(file_system)` — Wikipedia article +Descriptor for Lustre MDT/OST images backed by ldiskfs (ext4-compatible). The archive projection surfaces one target's backing-store namespace — NOT the distributed Lustre logical view, which requires correlating MDT metadata with objects striped across multiple OSTs. Maintenance is intentionally narrower than generic ext editing. Free-space wipe follows the ldiskfs block allocation bitmap, and shrink updates ext geometry in place, preserving every surviving allocated block. Add/replace/remove, defrag and structural relayout are not advertised because rebuilding through the generic ext writer would discard Lustre LMA/LOV/FID extended attributes and target metadata. Legacy "LUSTRE" / "LUst" object-header dumps remain inspection-only. Detection is extension-routed (.lustre / .ost / .mdt) and the legacy "LUSTRE" / "LUst" object-header magic at offset 0; ext4 superblock magic is deliberately NOT registered here because that would steal generic ext4 images. References: `https://wiki.lustre.org/Configuring_the_Lustre_File_System` — mkfs.lustre examples showing ldiskfs backing targets`https://wiki.lustre.org/Understanding_Lustre_Internals` — backing ldiskfs inspection and Lustre target internals`https://docs.kernel.org/filesystems/ext4/bitmaps.html` — ext4 allocation bitmap semantics and BLOCK_UNINIT warning`https://docs.kernel.org/filesystems/ext4/group_descr.html` — group descriptor and bitmap locations -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveFormatOperations`, `IArchiveShrinkable`, `IFormatDescriptor`, `ILayoutOptimizable`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -8670,8 +8725,11 @@ Implements `IArchiveFormatOperations`, `IFormatDescriptor`. | `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | | `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | +| `AnalyzeLayout` | `LayoutAnalysis AnalyzeLayout(Stream image)` | Reports the current ldiskfs block geometry. Structural relayout is deliberately not offered until a Lustre-aware writer can preserve all target metadata. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | | `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Shrinks an offline ldiskfs target by trimming trailing free blocks while preserving all surviving allocated blocks and Lustre-specific inode/xattr bytes. | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Zeroes complete ldiskfs blocks whose initialized allocation bitmap proves them free. Cluster-tip and deleted-dirent requests are deliberately ignored: those byte ranges cannot be proven dead without interpreting Lustre-specific inode/xattr semantics. | #### `LustreReader` @@ -9320,9 +9378,9 @@ One synthetic surface in the MooseFS master-metadata image: either the `metadata #### `MooseFsFormatDescriptor` -Partial R/O descriptor for MooseFS master-metadata images (`metadata.mfs`). Surfaces the metadata envelope (signature, counters, section index) and the raw payload bytes of each walked section. Path-tree (NODE/EDGE) and chunk-id (CHNK) bodies are version-specific and require golden samples to decode honestly — the reader makes no claim about their internal structure. MooseFS file content lives on chunk servers and is unreachable from a single metadata image. Listing therefore surfaces ONLY synthetic metadata + per-section raw payloads, never POSIX paths. References: `https://github.com/moosefs/moosefs` — canonical source (master metadata dump/load code)`https://moosefs.com/` — vendor site and documentation`https://en.wikipedia.org/wiki/Moose_File_System` — Wikipedia article +Descriptor for MooseFS master-metadata images (`metadata.mfs`). The standalone image contains namespace/chunk metadata, not file payloads; payload bytes live on chunk servers. The descriptor therefore exposes a synthetic forensic view of the metadata envelope rather than pretending a metadata dump is a self-contained MooseFS volume. Safe standalone-image maintenance is deliberately narrow: byte layout can be mapped, unused-space wiping fails closed (valid metadata dumps are tightly packed), and purge resets the image to MooseFS's official `MFSM NEW` empty bootstrap. Add/replace/remove, shrink and defragmentation require semantic NODE/EDGE/CHNK updates and/or live cluster coordination and are not advertised here. References: `https://github.com/moosefs/moosefs` — canonical implementation and metadata loader/store format`https://moosefs.com/` — vendor documentation -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchivePurgeable`, `IFormatDescriptor`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -9339,42 +9397,50 @@ Implements `IArchiveFormatOperations`, `IFormatDescriptor`. | `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | | `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the exact bytes occupied by a valid MooseFS metadata image. Unknown/truncated images return no extents so generic maintenance fails closed. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Extracts the synthetic metadata-image views. | +| `List` | `List List(Stream stream, string password)` | Lists the synthetic entries in the supplied metadata image. | +| `Purge` | `void Purge(Stream archive)` | Resets a valid metadata image to MooseFS's official empty bootstrap state. This only rewrites `metadata.mfs`; any chunk-server data is outside the image and therefore outside this operation. | #### `MooseFsReader` -Partial R/O reader for MooseFS master-metadata images (`metadata.mfs`). MooseFS is a fault-tolerant distributed FS — the master server keeps the namespace + chunk-server topology in a single binary metadata file, while file data lives on chunk servers. This reader understands the master metadata's outer envelope: 8-byte ASCII signature (e.g. `MFSM 2.0`, `MFSM 1.6`, `MFSM 1.5`, `MFSM 1.4`, `MFSM NEW`).For 1.6+ images: two 8-byte big-endian counters (file-id counter, metadata version) immediately after the signature.Sequence of sections. Each section: 8-byte ASCII type tag (`SESS 1.0`, `STAT 1.0`, `NODE 1.0`, `EDGE 1.0`, `FREE 1.0`, `XATR 1.0`, `CHNK 1.0`, `OPEN 1.0`, `FLCK 1.0`, `QUOT 1.0`, `ACLS 1.0`, …) + 8-byte big-endian payload length + that many payload bytes.Final 16-byte terminator `[MFS EOF MARKER]`. The reader walks the section index only — it does not attempt to decode NODE / EDGE record bodies, which differ between MooseFS minor versions and require ground-truth golden samples to validate. NODE/EDGE would give path tree + inode metadata; CHNK gives chunk-id mappings. None of those by themselves yield file content — MooseFS data lives on chunk servers and is only reachable via the live MooseFS protocol. Therefore the reader exposes: `metadata.ini` — human-readable summary of header + section table (name, payload offset, payload length).`moosefs-master.bin` — the raw image, byte-for-byte.`section_.bin` — the raw payload bytes of each section the index walk surfaced (NODE, EDGE, CHNK, …). Useful for offline forensics; we make no claim about their internal structure. If section-walk fails (signature past the 8-byte tag is not recognised, a section length runs past EOF, the EOF marker is missing, …), the reader falls back to a header-only surface (metadata.ini + raw) and records the parse failure in `metadata.ini`'s `parse_status` field. This is the honest "we recognise the envelope but couldn't walk the contents" mode rather than silently inventing entries. +Partial reader for MooseFS master-metadata images (`metadata.mfs`). It understands the versioned outer envelope and section framing, but does not decode the version-specific NODE / EDGE / CHNK bodies or contact chunk servers. Consequently all surfaced archive entries are synthetic forensic views of the metadata image rather than the mounted MooseFS namespace. Implements `IDisposable`. | Member | Signature | Summary | | --- | --- | --- | -| `MooseFsReader` | `MooseFsReader(Stream stream)` | Initializes a new instance of `MooseFsReader`. | -| `EofMarker` | `static readonly byte[] EofMarker` | 16-byte MooseFS end-of-file marker following the last section. | -| `MasterTag` | `static readonly byte[] MasterTag` | MooseFS master metadata 4-byte prefix: ASCII "MFSM". | -| `Entries` | `IReadOnlyList Entries { get; }` | Listing of every entry this image surfaces. | -| `FileIdCounter` | `ulong? FileIdCounter { get; }` | File-id counter from the modern (1.6+) post-signature header, or `null` when the image is too short or pre-1.6. | -| `MetadataVersion` | `ulong? MetadataVersion { get; }` | Metadata version counter from the modern (1.6+) post-signature header, or `null` when the image is too short or pre-1.6. | -| `ParseStatus` | `string ParseStatus { get; }` | Human-readable description of how the section walk terminated: `"ok"` (full walk + EOF marker), `"truncated"` (section walk stopped before EOF marker), `"header-only"` (image too short for any sections), or `"unsupported-header"` (no MFSM tag). | -| `Sections` | `IReadOnlyList Sections { get; }` | Section index walked from the master metadata stream. | -| `Signature` | `string Signature { get; }` | The 8-byte ASCII signature at offset 0 (e.g. `"MFSM 2.0"`). | -| `ValidHeader` | `bool ValidHeader { get; }` | True when the 4-byte `MFSM` tag was present at offset 0. | +| `MooseFsReader` | `MooseFsReader(Stream stream)` | Initializes a reader over one MooseFS metadata image. | +| `EofMarker` | `static readonly byte[] EofMarker` | Modern (1.6+) 16-byte MooseFS end-of-file marker. | +| `MasterTag` | `static readonly byte[] MasterTag` | MooseFS master metadata 4-byte prefix: ASCII `MFSM`. | +| `Entries` | `IReadOnlyList Entries { get; }` | Listing of every synthetic entry surfaced from this image. | +| `FileFormatVersion` | `byte? FileFormatVersion { get; }` | Packed file-format version (`0x16` for 1.6, `0x20` for 2.0), or `null` for the special `MFSM NEW` bootstrap image. | +| `FileIdCounter` | `ulong? FileIdCounter { get; }` | Compatibility view of the first eight bytes after the signature, matching the value older CompressionWorkbench builds exposed under this incorrect name. MooseFS does not define this field as a file-id counter. | +| `ImageSize` | `long ImageSize { get; }` | The exact image size consumed by this reader. | +| `IsEmptyBootstrap` | `bool IsEmptyBootstrap { get; }` | Whether this is MooseFS's official eight-byte empty bootstrap image. | +| `MaxNodeId` | `uint? MaxNodeId { get; }` | Maximum node id from the pre-2.0 metadata header. Not present in 2.0+. | +| `MetaId` | `ulong? MetaId { get; }` | Metadata instance id stored by 2.0+ images. | +| `MetadataVersion` | `ulong? MetadataVersion { get; }` | Metadata/changelog version stored in the master metadata header. | +| `NextSessionId` | `uint? NextSessionId { get; }` | Next session id from the pre-2.0 metadata header. Not present in 2.0+. | +| `ParseStatus` | `string ParseStatus { get; }` | Human-readable parse result: `ok`, `header-only`, `truncated`, `trailing-data`, or `unsupported-header`. | +| `Sections` | `IReadOnlyList Sections { get; }` | Section index walked from section-framed (1.6+) metadata. | +| `Signature` | `string Signature { get; }` | The 8-byte ASCII signature, for example `MFSM 2.0`. | +| `ValidHeader` | `bool ValidHeader { get; }` | True once the `MFSM` prefix has been verified. | | `Dispose` | `void Dispose()` | Releases resources held by this instance. | -| `Extract` | `byte[] Extract(MooseFsEntry entry)` | Returns the bytes that back the given entry (in-memory). | +| `Extract` | `byte[] Extract(MooseFsEntry entry)` | Returns the bytes backing a surfaced synthetic entry. | #### `MooseFsReader.SectionEntry` -One walked section from the master metadata stream. +One walked section from a 1.6+ master metadata stream. Implements `IEquatable`. | Member | Signature | Summary | | --- | --- | --- | -| `SectionEntry` | `SectionEntry(string Tag, long Offset, long Length)` | One walked section from the master metadata stream. | -| `Length` | `long Length { get; init; }` | Length of the section payload in bytes. | -| `Offset` | `long Offset { get; init; }` | Byte offset of the section payload (after the 16-byte tag+length). | -| `Tag` | `string Tag { get; init; }` | The 8-byte ASCII tag (e.g. `"NODE 1.0"`), trimmed. | +| `SectionEntry` | `SectionEntry(string Tag, long Offset, long Length)` | One walked section from a 1.6+ master metadata stream. | +| `Length` | `long Length { get; init; }` | Payload length in bytes. | +| `Offset` | `long Offset { get; init; }` | Byte offset of the section payload. | +| `Tag` | `string Tag { get; init; }` | Eight-byte section tag, for example `NODE 1.0`. | ### Namespace `FileSystem.Msa` @@ -10174,27 +10240,37 @@ Writes a NetWare 386 disk image: a partition table naming one NetWare partition, #### `Nwfs386FormatDescriptor` -Read-only descriptor for Novell NetWare 386 (NWFS386) raw partition dumps, detected via the "NetW" ASCII prefix at offset 0. DOS partition type `0x65`. The on-disk format is FAT-like but proprietary; no parser is attempted — the image is surfaced as a single opaque entry with metadata.ini noting the partition-type hint. References: `https://www.win.tue.nl/~aeb/partitions/partition_types-1.html` — partition-type catalogue (0x65 = Novell NetWare)`https://en.wikipedia.org/wiki/NetWare_File_System` — Wikipedia articleNovell NetWare 386 internal documentation — the on-disk format was never published +Compatibility descriptor for the Novell NetWare 386 Traditional File System. The actual reader/writer lives in `FileSystem.Nwfs`; this descriptor keeps the historical `Nwfs386` id/extensions while sharing that implementation. -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveInMemoryExtract`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `IFormatOptionsSchema`, `ILayoutOptimizable`. | Member | Signature | Summary | | --- | --- | --- | | `Nwfs386FormatDescriptor` | `Nwfs386FormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `OptionsSchema` | `IReadOnlyList OptionsSchema { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | | +| `AnalyzeLayout` | `LayoutAnalysis AnalyzeLayout(Stream image)` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Defragment` | `void Defragment(Stream archive)` | | +| `ExtractEntry` | `void ExtractEntry(Stream input, string entryName, Stream output, string password)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `OpenEntry` | `Stream OpenEntry(Stream archive, string entryName, string password)` | | +| `RebuildStreaming` | `void RebuildStreaming(Stream source, Stream target, LayoutRebuildOptions options)` | | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | | +| `Shrink` | `void Shrink(Stream input, Stream output)` | | ### Namespace `FileSystem.Ocfs2` @@ -10410,45 +10486,52 @@ Represents an one fs entry. #### `OneFsFormatDescriptor` -Stage 0 detection-only descriptor for Dell EMC Isilon OneFS LIN-tree root images. Surfaces only a synthetic `metadata.ini` and the raw image bytes; no real file-walk is attempted. Why R/O promotion is impossible (per CONTRIBUTING.md promotion gates):No single-image content surface. OneFS is a clustered scale-out NAS — every file is split into "protection groups" striped across drives and nodes with FEC (Forward Error Correction, N+M:B layout, e.g. N+2:1). A single drive/node image carries only one stripe; the file data cannot be reconstructed without the peer nodes. A read-only reader from one image can never return correct file bytes. LIN tree is cluster-wide. The Logical Inode Number tree (the OneFS metadata index) lives across nodes, not in a single superblock. There is no per-image inode-to-block mapping to walk. Proprietary on-disk format, no public specification. Dell EMC has never published the OneFS on-disk format. No open-source reverse-engineered reader exists. Without a spec we cannot honour the CONTRIBUTING rule "never advertise capabilities you cannot prove against a real spec". FreeBSD/UFS ancestry does NOT give us a UFS reader fallback. OneFS runs on a FreeBSD-derived kernel, but the filesystem layer is entirely proprietary — it is NOT FFS/UFS at the on-disk level. UFS1 places its superblock magic `0x00011954` at offset 8192; OneFS images have the ASCII `"OneFS"` tag at offset 0 and no UFS superblock. Routing OneFS images through `UfsReader` would fail the magic check and (if forced) return arbitrary bytes — the textbook mutual-compensation trap. Conclusion: Stage-0 detection only. Surface the magic, raw bytes, and a `metadata.ini` documenting the limitation. R/O promotion is blocked on (a) Dell EMC publishing the spec and (b) a multi-node ingest path — neither is in reach. References: Dell EMC "PowerScale OneFS Technical Overview" whitepaper — high-level architecture only; no on-disk spec is published`https://en.wikipedia.org/wiki/OneFS_distributed_file_system` — Wikipedia article +Conservative inspection descriptor for Dell PowerScale / Isilon OneFS media. -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveFormatOperations`, `IFormatDescriptor`, `ILayoutOptimizable`. | Member | Signature | Summary | | --- | --- | --- | | `OneFsFormatDescriptor` | `OneFsFormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the conservative inspection capabilities. Integrity testing is not advertised: without a verified superblock/tree parser, successfully copying an opaque image does not prove that the filesystem is structurally sound. | +| `Category` | `FormatCategory Category { get; }` | Gets the format category. | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets compound extensions. | +| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the conventional extension for explicitly supplied raw media. | +| `Description` | `string Description { get; }` | Gets a description of the intentionally limited inspection surface. | +| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | +| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the recognized extensions. | +| `Family` | `AlgorithmFamily Family { get; }` | Gets the algorithm family. | +| `Id` | `string Id { get; }` | Gets the registry id. | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets fixed magic signatures. Dell does not publish an authoritative offset-zero raw-media signature, so OneFS intentionally has none. | +| `Methods` | `IReadOnlyList Methods { get; }` | Gets the pseudo-archive storage method. | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | +| `AnalyzeLayout` | `LayoutAnalysis AnalyzeLayout(Stream image)` | Reports the fixed physical geometry documented by Dell without reading or interpreting proprietary allocation metadata. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Extracts selected inspection entries through bounded streams. | +| `List` | `List List(Stream stream, string password)` | Lists the two conservative inspection entries without reading the image payload. | #### `OneFsReader` -Stage 0 detection-only reader for Dell EMC Isilon OneFS LIN-tree root images. OneFS is a clustered scale-out NAS — its single-image surface is the LIN-tree root block, whose first bytes are the ASCII tag `"OneFS"` (5 bytes, 0x4F 0x6E 0x65 0x46 0x53) or the short `"ONEF"` tag (0x4F 0x4E 0x45 0x46 = 0x4F4E4546 BE int) used in some node-local boot images. Only the tag is verified; the real LIN tree (logical inode number tree) is a cluster-wide construct and cannot be walked from a single image. File data is FEC-striped across nodes (N+M:B protection groups) — even a complete single-drive image carries only one stripe and cannot reconstruct file content without peer nodes. OneFS shares OS ancestry with FreeBSD, but the on-disk filesystem layer is proprietary and NOT UFS-compatible: there is no UFS1 superblock magic (`0x00011954`) at the UFS1 superblock offset (8192). The OneFS on-disk format has never been publicly specified by Dell EMC. +Conservative single-image inspection surface for Dell PowerScale / Isilon OneFS media. Implements `IDisposable`. | Member | Signature | Summary | | --- | --- | --- | -| `OneFsReader` | `OneFsReader(Stream stream)` | Initializes a new instance of `OneFsReader`. | -| `LongTag` | `static readonly byte[] LongTag` | OneFS LIN-tree-root long tag: ASCII "OneFS" (5 bytes). | -| `ShortTag` | `static readonly byte[] ShortTag` | OneFS short tag: ASCII "ONEF" (4 bytes, 0x4F4E4546 BE). | -| `Entries` | `IReadOnlyList Entries { get; }` | Gets the entries. | -| `Tag` | `string Tag { get; }` | Gets or sets the tag. | -| `TrailingWord` | `uint TrailingWord { get; }` | Gets or sets the trailing word. | -| `ValidHeader` | `bool ValidHeader { get; }` | Gets a value indicating whether valid header. | -| `Dispose` | `void Dispose()` | Releases resources held by this instance. | -| `Extract` | `byte[] Extract(OneFsEntry entry)` | Decodes the supplied input. | +| `OneFsReader` | `OneFsReader(Stream stream)` | Initializes a conservative OneFS image reader without consuming or owning the source stream. Kept as the original one-argument public constructor for binary/source API compatibility. | +| `OneFsReader` | `OneFsReader(Stream stream, bool leaveOpen)` | Initializes a conservative OneFS image reader without consuming the image. | +| `BlocksPerCylinderGroup` | `const int BlocksPerCylinderGroup` | Documented number of 8 KiB blocks in a 32 MiB cylinder group. | +| `CylinderGroupSize` | `const int CylinderGroupSize` | Documented physical cylinder-group size of each OneFS data disk. | +| `MetadataEntryName` | `const string MetadataEntryName` | Name of the synthetic inspection metadata entry. | +| `PhysicalBlockSize` | `const int PhysicalBlockSize` | Documented OneFS filesystem block size. | +| `RawImageEntryName` | `const string RawImageEntryName` | Name of the opaque raw-image entry. | +| `Entries` | `IReadOnlyList Entries { get; }` | Gets the synthetic entries exposed by the conservative reader. | +| `ImageSize` | `long ImageSize { get; }` | Gets the byte length of the opaque source image. | +| `Tag` | `string Tag { get; }` | Legacy compatibility property. No authoritative fixed OneFS raw-media tag is currently known, so this value is always empty. | +| `TrailingWord` | `uint TrailingWord { get; }` | Legacy compatibility property. No undocumented trailing header word is interpreted; this value is always zero. | +| `ValidHeader` | `bool ValidHeader { get; }` | Legacy compatibility property. The current reader does not claim to have validated a proprietary raw-media header, so this value is always false. | +| `Dispose` | `void Dispose()` | Releases the source stream when ownership was requested. | +| `Extract` | `byte[] Extract(OneFsEntry entry)` | Materializes an entry in memory. Prefer `OpenEntry` for the raw image so large media remains streaming. | +| `OpenEntry` | `Stream OpenEntry(OneFsEntry entry)` | Opens an entry as a bounded read-only stream. Opening the raw image resets the source to byte zero but never copies the payload into managed memory. | ### Namespace `FileSystem.OpenVms` @@ -11446,7 +11529,7 @@ WORM writer for QNX6 (Neutrino) filesystem images. Emits a power-safe layout: th ### Namespace `FileSystem.Refs` -[`RefsBlockMover`](#refsblockmover) · [`RefsExtentMap`](#refsextentmap) · [`RefsFormatDescriptor`](#refsformatdescriptor) +[`RefsBlockMover`](#refsblockmover) · [`RefsExtentMap`](#refsextentmap) · [`RefsFormatDescriptor`](#refsformatdescriptor) · [`RefsOfflineBlockCloner`](#refsofflineblockcloner) #### `RefsBlockMover` @@ -11507,6 +11590,14 @@ Implements `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveModifi | `ProbeFilesystem` | `FilesystemDriverProfile ProbeFilesystem(Stream image)` | Probes the image and reports the filesystem driver profile. | | `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes regular files or empty directories from an unmounted ReFS image. Namespace deletion is published through immutable B+ replacement pages and the alternate CHKP. | +#### `RefsOfflineBlockCloner` + +Performs the bounded offline ReFS block-clone operation for two existing, equal-sized ordinary extent-backed files. The destination's stream mapping is replaced by the source mapping through immutable object/Object-Table CoW, while root #6 gains the corresponding shared-cluster reference counts in the same alternate-checkpoint publication. This is deliberately narrower than FSCTL_DUPLICATE_EXTENTS_TO_FILE: it clones a whole cluster-aligned file only, does not create names or extend EOF, and refuses resident, sparse, integrity, already-shared and otherwise undecoded stream layouts. Native mounted-driver/MLog semantics remain fail-closed. + +| Member | Signature | Summary | +| --- | --- | --- | +| `CloneWholeFile` | `static void CloneWholeFile(Stream image, string sourcePath, string destinationPath)` | Replaces the contents of an existing destination file with a metadata-only clone of an existing source file on an unmounted ReFS image. | + ### Namespace `FileSystem.Reiser4` [`Reiser4BlockMover`](#reiser4blockmover) · [`Reiser4FormatDescriptor`](#reiser4formatdescriptor) · [`Reiser4MasterSb`](#reiser4mastersb) · [`Reiser4Reader`](#reiser4reader) · [`Reiser4Reader.Entry`](#reiser4readerentry) · [`Reiser4Writer`](#reiser4writer) @@ -13291,9 +13382,9 @@ Represents a tux 3 entry. #### `Tux3FormatDescriptor` -Read-only native-superblock descriptor for the linux-tux3 research filesystem. +Native-superblock descriptor for the linux-tux3 research filesystem. -Implements `IArchiveFormatOperations`, `IFormatDescriptor`, `ISyntheticEntryNames`. +Implements `IArchiveFormatOperations`, `IArchiveShrinkable`, `IFilesystemExtentMap`, `IFormatDescriptor`, `ISyntheticEntryNames`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -13311,8 +13402,10 @@ Implements `IArchiveFormatOperations`, `IFormatDescriptor`, `ISyntheticEntryName | `Methods` | `IReadOnlyList Methods { get; }` | | | `SyntheticEntryNames` | `IReadOnlySet SyntheticEntryNames { get; }` | | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `EnumerateExtents` | `IEnumerable EnumerateExtents(Stream image)` | Enumerates the provable byte layout without guessing at undecoded TUX3 allocation state. The declared native volume is reserved wholesale; only trailing bytes outside it are free. Invalid or arithmetically unrepresentable volume metadata reserves the complete physical image. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | | `List` | `List List(Stream stream, string password)` | | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Removes only bytes beyond the volume size declared by `volblocks * blocksize`. Malformed, truncated, or arithmetically invalid images are copied through unchanged. | #### `Tux3Reader` @@ -14147,7 +14240,7 @@ Implements `IDisposable`. ### Namespace `FileSystem.Xfs` -[`XfsBlockMover`](#xfsblockmover) · [`XfsEntry`](#xfsentry) · [`XfsExtentMap`](#xfsextentmap) · [`XfsFilesystemDriverAdapter`](#xfsfilesystemdriveradapter) · [`XfsFormatDescriptor`](#xfsformatdescriptor) · [`XfsInPlaceAdder`](#xfsinplaceadder) · [`XfsModifier`](#xfsmodifier) · [`XfsReader`](#xfsreader) · [`XfsWriter`](#xfswriter) +[`XfsBlockMover`](#xfsblockmover) · [`XfsEntry`](#xfsentry) · [`XfsExtendedAttributes`](#xfsextendedattributes) · [`XfsExtentMap`](#xfsextentmap) · [`XfsFilesystemDriverAdapter`](#xfsfilesystemdriveradapter) · [`XfsFormatDescriptor`](#xfsformatdescriptor) · [`XfsInPlaceAdder`](#xfsinplaceadder) · [`XfsModifier`](#xfsmodifier) · [`XfsReader`](#xfsreader) · [`XfsWriter`](#xfswriter) #### `XfsBlockMover` @@ -14179,6 +14272,16 @@ Represents a xfs entry. | `Name` | `string Name { get; init; }` | Gets or sets the name. | | `Size` | `long Size { get; init; }` | Gets or sets the size. | +#### `XfsExtendedAttributes` + +Reads and mutates XFS short-form extended attributes stored in an inode's attribute fork. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Read` | `static IReadOnlyDictionary Read(Stream image, string path)` | Reads all short-form xattrs for `path`. | +| `Remove` | `static bool Remove(Stream image, string path, string name)` | Removes one short-form xattr, returning false when it was absent. | +| `Set` | `static void Set(Stream image, string path, string name, ReadOnlySpan value)` | Creates or replaces a short-form xattr. If the resulting fork does not fit in the inode, the call fails without modifying the image. | + #### `XfsExtentMap` Walks an XFS image and yields its actual on-disk byte layout. Targets the WORM writer profile: per-AG superblock + AGF + AGI + AGFL + bnobt/cntbt/ inobt headers as MetadataReserved, plus per-file extents (BMBT_REC packed 128-bit format) as Used runs. For inodes whose data fork is in `local` (inline) format, the file content lives inside the inode itself and surfaces as MetadataReserved. Streaming: never loads the whole image. All reads flow through a `SectorCache` so multi-TB XFS images (an XFS volume can span thousands of AGs) work without OOM.