diff --git a/.github/workflows/refs-diagnostic.yml b/.github/workflows/refs-diagnostic.yml new file mode 100644 index 000000000..3470908a9 --- /dev/null +++ b/.github/workflows/refs-diagnostic.yml @@ -0,0 +1,57 @@ +name: ReFS diagnostic + +on: + push: + branches: [refs-block-refcount-clone] + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Clone PNGCrushCS sibling + run: git clone --depth 1 https://github.com/Hawkynt/PNGCrushCS.git "$GITHUB_WORKSPACE/../PNGCrushCS" + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + dotnet-quality: preview + - name: Pre-build sibling + shell: bash + run: | + set -euo pipefail + cd "$GITHUB_WORKSPACE/../PNGCrushCS" + REQUIRED="Compression.Core FileFormat.Core FileFormat.Bmp FileFormat.Apng FileFormat.Tiff FileFormat.BigTiff FileFormat.Mng FileFormat.Fli FileFormat.Dcx FileFormat.Icns FileFormat.Mpo FileFormat.Jpeg FileFormat.Png" + locate() { + local proj="$1" + if [ -f "$proj/$proj.csproj" ]; then echo "$proj"; return 0; fi + if [ -f "FileFormats/$proj/$proj.csproj" ]; then echo "FileFormats/$proj"; return 0; fi + return 1 + } + for proj in $REQUIRED; do + if ! locate "$proj" >/dev/null; then exit 0; fi + done + for proj in $REQUIRED; do + dir="$(locate "$proj")" + dotnet build "$dir/$proj.csproj" --configuration Release --nologo + done + - name: Restore + run: dotnet restore CompressionWorkbench.slnx + - name: Build and capture diagnostics + id: build + shell: bash + continue-on-error: true + run: | + set -o pipefail + dotnet build CompressionWorkbench.slnx --configuration Release --no-restore 2>&1 | tee refs-build.log + - uses: actions/upload-artifact@v4 + with: + name: refs-build-log + path: refs-build.log + - name: Propagate build result + if: steps.build.outcome != 'success' + run: exit 1 diff --git a/.github/workflows/regenerate-moosefs-docs.yml b/.github/workflows/regenerate-moosefs-docs.yml new file mode 100644 index 000000000..d18555b40 --- /dev/null +++ b/.github/workflows/regenerate-moosefs-docs.yml @@ -0,0 +1,51 @@ +name: Regenerate MooseFS docs + +on: + push: + branches: [feature/moosefs-maintenance] + +concurrency: + group: regenerate-moosefs-docs + cancel-in-progress: true + +permissions: + contents: write + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + +jobs: + regenerate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + dotnet-quality: 'preview' + - name: Rewrite filesystem support matrix + env: + CWB_WRITE_DOCS: '1' + run: >- + dotnet test Compression.Tests/Compression.Tests.csproj + --configuration Release + --filter FullyQualifiedName~FilesystemReadmeIsCurrentTests.TheSupportMatrix_SaysWhatTheCodeSays + --nologo + - name: Rewrite filesystem package API reference + uses: Hawkynt/RepositoryTemplate/package-readme@v1 + with: + mode: write + project: Hawkynt.FileFormats.FileSystems/Hawkynt.FileFormats.FileSystems.csproj + - name: Commit generated documentation + shell: bash + run: | + set -euo pipefail + if git diff --quiet -- Hawkynt.FileFormats.FileSystems/README.md Hawkynt.FileFormats.FileSystems/REFERENCE.md; then + echo 'Filesystem package documentation already current.' + exit 0 + fi + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add Hawkynt.FileFormats.FileSystems/README.md Hawkynt.FileFormats.FileSystems/REFERENCE.md + git commit -m '* regenerate filesystem package documentation' + git push diff --git a/Compression.Core/DiskImage/PartitionedDiskLister.cs b/Compression.Core/DiskImage/PartitionedDiskLister.cs index 601965ba1..f64a1b4d9 100644 --- a/Compression.Core/DiskImage/PartitionedDiskLister.cs +++ b/Compression.Core/DiskImage/PartitionedDiskLister.cs @@ -175,7 +175,7 @@ private static string MakePartitionPrefix(PartitionEntry part) { // ── Add / Remove (partition-aware) ──────────────────────────────────── /// - /// Partition-aware . Returns true + /// Partition-aware . Returns true /// when a partition table was present and at least one input was dispatched /// through the partition-aware path; false when no partition table /// exists so the caller can fall through to the existing single-FS path. @@ -243,7 +243,7 @@ public static bool TryAdd(Stream disk, IReadOnlyList inputs) { } /// - /// Partition-aware . Returns true + /// Partition-aware . Returns true /// when a partition table was present and at least one entry was removed; /// false if no partition table was detected. Entry name shapes: /// Partition<N>_<Type>/<inner> deletes an inner-FS diff --git a/Compression.Core/README.md b/Compression.Core/README.md index a5c5465c1..82ec5cb56 100644 --- a/Compression.Core/README.md +++ b/Compression.Core/README.md @@ -204,7 +204,7 @@ Use the concrete version you intend to consume; this document does not predict a -Every public and protected member of all 627 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Compression.Core/REFERENCE.md). +Every public and protected member of all 628 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Compression.Core/REFERENCE.md). diff --git a/Compression.Core/REFERENCE.md b/Compression.Core/REFERENCE.md index 1f7348efc..b5c55a7ca 100644 --- a/Compression.Core/REFERENCE.md +++ b/Compression.Core/REFERENCE.md @@ -6295,7 +6295,7 @@ Run-Length Encoding (RLE) transform. Encodes runs of identical bytes as (count, ### Namespace `Compression.Registry` -[`AlgorithmFamily`](#algorithmfamily) · [`ApeTagReader`](#apetagreader) · [`ApeTagReader.ApeTag`](#apetagreaderapetag) · [`ArchiveEntryInfo`](#archiveentryinfo) · [`ArchiveInputInfo`](#archiveinputinfo) · [`ArchiveShrinker`](#archiveshrinker) · [`AudioEncodedStream`](#audioencodedstream) · [`AudioPacket`](#audiopacket) · [`AudioPcmBuffer`](#audiopcmbuffer) · [`AudioPcmEncoding`](#audiopcmencoding) · [`AudioPcmFormat`](#audiopcmformat) · [`AudioPseudoArchive`](#audiopseudoarchive) · [`AudioPseudoArchive.Entry`](#audiopseudoarchiveentry) · [`AudioStreamFormat`](#audiostreamformat) · [`BlockDeviceGeometry`](#blockdevicegeometry) · [`BlockDeviceStream`](#blockdevicestream) · [`BuildingBlockRegistry`](#buildingblockregistry) · [`CompoundTarDescriptor`](#compoundtardescriptor) · [`DefragBlockClass`](#defragblockclass) · [`DefragBlockInfo`](#defragblockinfo) · [`DefragBlockKind`](#defragblockkind) · [`DefragContentGuard`](#defragcontentguard) · [`DefragMode`](#defragmode) · [`DefragOptions`](#defragoptions) · [`DefragProgressEvent`](#defragprogressevent) · [`DefragRebuilder`](#defragrebuilder) · [`EntropyDetector`](#entropydetector) · [`FatDirStamp`](#fatdirstamp) · [`FilesystemCompressionParameter`](#filesystemcompressionparameter) · [`FilesystemCompressionProfile`](#filesystemcompressionprofile) · [`FilesystemDirectoryEntry`](#filesystemdirectoryentry) · [`FilesystemDriverBindingKind`](#filesystemdriverbindingkind) · [`FilesystemDriverCapabilities`](#filesystemdrivercapabilities) · [`FilesystemDriverCoverage`](#filesystemdrivercoverage) · [`FilesystemDriverDerivation`](#filesystemdriverderivation) · [`FilesystemDriverProfile`](#filesystemdriverprofile) · [`FilesystemDriverReadinessLayer`](#filesystemdriverreadinesslayer) · [`FilesystemDriverReadinessReport`](#filesystemdriverreadinessreport) · [`FilesystemDriverTarget`](#filesystemdrivertarget) · [`FilesystemMetadataPatch`](#filesystemmetadatapatch) · [`FilesystemMutationModel`](#filesystemmutationmodel) · [`FilesystemNodeId`](#filesystemnodeid) · [`FilesystemNodeInfo`](#filesystemnodeinfo) · [`FilesystemNodeKind`](#filesystemnodekind) · [`FilesystemOpenOptions`](#filesystemopenoptions) · [`FilesystemOptimization`](#filesystemoptimization) · [`FilesystemOptimizationAdapters`](#filesystemoptimizationadapters) · [`FilesystemOptimizationAdapters.HardLinkDeduplicator`](#filesystemoptimizationadaptershardlinkdeduplicator) · [`FilesystemOptimizationAdapters.SymbolicLinkDeduplicator`](#filesystemoptimizationadapterssymboliclinkdeduplicator) · [`FilesystemOptimizationExtensions`](#filesystemoptimizationextensions) · [`FilesystemOptimizationFeatures`](#filesystemoptimizationfeatures) · [`FilesystemOptimizationOptions`](#filesystemoptimizationoptions) · [`FilesystemSchemaPresets`](#filesystemschemapresets) · [`FilesystemSnapshotDirectoryEntry`](#filesystemsnapshotdirectoryentry) · [`FilesystemSnapshotNode`](#filesystemsnapshotnode) · [`FormatCapabilities`](#formatcapabilities) · [`FormatCategory`](#formatcategory) · [`FormatCreateOptions`](#formatcreateoptions) · [`FormatDetectionSignature`](#formatdetectionsignature) · [`FormatHeaderMatch`](#formatheadermatch) · [`FormatHealth`](#formathealth) · [`FormatHelpers`](#formathelpers) · [`FormatMethodInfo`](#formatmethodinfo) · [`FormatOptionDescriptor`](#formatoptiondescriptor) · [`FormatOptionKind`](#formatoptionkind) · [`FormatRegistry`](#formatregistry) · [`HardLinkDeduplicationSemantics`](#hardlinkdeduplicationsemantics) · [`IArchiveCreatable`](#iarchivecreatable) · [`IArchiveDefragmentable`](#iarchivedefragmentable) · [`IArchiveFormatOperations`](#iarchiveformatoperations) · [`IArchiveInMemoryExtract`](#iarchiveinmemoryextract) · [`IArchiveLayoutMap`](#iarchivelayoutmap) · [`IArchiveModifiable`](#iarchivemodifiable) · [`IArchivePurgeable`](#iarchivepurgeable) · [`IArchiveShrinkable`](#iarchiveshrinkable) · [`IArchiveWriteConstraints`](#iarchivewriteconstraints) · [`IAsyncArchiveOperations`](#iasyncarchiveoperations) · [`IAudioContainerFormat`](#iaudiocontainerformat) · [`IAudioDemuxSource`](#iaudiodemuxsource) · [`IAudioMuxTarget`](#iaudiomuxtarget) · [`IAudioPcmSource`](#iaudiopcmsource) · [`IAudioPcmTarget`](#iaudiopcmtarget) · [`IBlockDeviceFilesystemDriverProvider`](#iblockdevicefilesystemdriverprovider) · [`IBlockDeviceProvider`](#iblockdeviceprovider) · [`IBuildingBlock`](#ibuildingblock) · [`IContainerRemuxable`](#icontainerremuxable) · [`IFileInternalChunkMover`](#ifileinternalchunkmover) · [`IFileInternalLayoutMap`](#ifileinternallayoutmap) · [`IFilesystemBlockMover`](#ifilesystemblockmover) · [`IFilesystemDriverAdapter`](#ifilesystemdriveradapter) · [`IFilesystemDriverProvider`](#ifilesystemdriverprovider) · [`IFilesystemDriverReadinessProvider`](#ifilesystemdriverreadinessprovider) · [`IFilesystemExtentMap`](#ifilesystemextentmap) · [`IFilesystemFileHandle`](#ifilesystemfilehandle) · [`IFilesystemMetadataMover`](#ifilesystemmetadatamover) · [`IFilesystemPlaceable`](#ifilesystemplaceable) · [`IFilesystemScrambleable`](#ifilesystemscrambleable) · [`IFilesystemSession`](#ifilesystemsession) · [`IFilesystemTransaction`](#ifilesystemtransaction) · [`IFormatDescriptor`](#iformatdescriptor) · [`IFormatDetectionSource`](#iformatdetectionsource) · [`IFormatOptionsSchema`](#iformatoptionsschema) · [`IFormatValidator`](#iformatvalidator) · [`ILayoutOptimizable`](#ilayoutoptimizable) · [`IPartitionEditable`](#ipartitioneditable) · [`IRandomAccessBlockDevice`](#irandomaccessblockdevice) · [`IRandomAccessBlockDeviceProvider`](#irandomaccessblockdeviceprovider) · [`IRawTrackDevice`](#irawtrackdevice) · [`IRawTrackDeviceProvider`](#irawtrackdeviceprovider) · [`IStreamFormatOperations`](#istreamformatoperations) · [`ISymbolicLinkDeduplicationLayout`](#isymboliclinkdeduplicationlayout) · [`ISyntheticEntryNames`](#isyntheticentrynames) · [`IWipeEmpty`](#iwipeempty) · [`InnerFsDetector`](#innerfsdetector) · [`IssueSeverity`](#issueseverity) · [`LayoutAnalysis`](#layoutanalysis) · [`LayoutPatch`](#layoutpatch) · [`LayoutProfile`](#layoutprofile) · [`LayoutRebuildOptions`](#layoutrebuildoptions) · [`LayoutReclaim`](#layoutreclaim) · [`MagicSignature`](#magicsignature) · [`MediaProfile`](#mediaprofile) · [`MediaProfileLookup`](#mediaprofilelookup) · [`MetadataPlacementProfile`](#metadataplacementprofile) · [`MetadataPlacementRule`](#metadataplacementrule) · [`MetadataZone`](#metadatazone) · [`MethodNameParser`](#methodnameparser) · [`ModifyRebuilder`](#modifyrebuilder) · [`MutableRebuildFilesystemSession`](#mutablerebuildfilesystemsession) · [`MutableRebuildFilesystemSession.RebuildImage`](#mutablerebuildfilesystemsessionrebuildimage) · [`MutableRebuildFilesystemSession.ValidateImage`](#mutablerebuildfilesystemsessionvalidateimage) · [`PartitionBlockDevice`](#partitionblockdevice) · [`PlacementOptions`](#placementoptions) · [`PlacementZone`](#placementzone) · [`RawDiskShrinkRebuilder`](#rawdiskshrinkrebuilder) · [`RawTrackInfo`](#rawtrackinfo) · [`ReadOnlyFilesystemSnapshotSession`](#readonlyfilesystemsnapshotsession) · [`RebuildFilesystemEntry`](#rebuildfilesystementry) · [`RebuildVerb`](#rebuildverb) · [`ScrambleOptions`](#scrambleoptions) · [`SpoolingReadOnlyFileHandle`](#spoolingreadonlyfilehandle) · [`StreamBlockDevice`](#streamblockdevice) · [`SymlinkResolver`](#symlinkresolver) · [`UnusedSpaceWiper`](#unusedspacewiper) · [`ValidationIssue`](#validationissue) · [`ValidationLevel`](#validationlevel) · [`ValidationResult`](#validationresult) · [`WholeImageRebuildCommitter`](#wholeimagerebuildcommitter) +[`AlgorithmFamily`](#algorithmfamily) · [`ApeTagReader`](#apetagreader) · [`ApeTagReader.ApeTag`](#apetagreaderapetag) · [`ArchiveEntryInfo`](#archiveentryinfo) · [`ArchiveInputInfo`](#archiveinputinfo) · [`ArchiveMutationOptions`](#archivemutationoptions) · [`ArchiveShrinker`](#archiveshrinker) · [`AudioEncodedStream`](#audioencodedstream) · [`AudioPacket`](#audiopacket) · [`AudioPcmBuffer`](#audiopcmbuffer) · [`AudioPcmEncoding`](#audiopcmencoding) · [`AudioPcmFormat`](#audiopcmformat) · [`AudioPseudoArchive`](#audiopseudoarchive) · [`AudioPseudoArchive.Entry`](#audiopseudoarchiveentry) · [`AudioStreamFormat`](#audiostreamformat) · [`BlockDeviceGeometry`](#blockdevicegeometry) · [`BlockDeviceStream`](#blockdevicestream) · [`BuildingBlockRegistry`](#buildingblockregistry) · [`CompoundTarDescriptor`](#compoundtardescriptor) · [`DefragBlockClass`](#defragblockclass) · [`DefragBlockInfo`](#defragblockinfo) · [`DefragBlockKind`](#defragblockkind) · [`DefragContentGuard`](#defragcontentguard) · [`DefragMode`](#defragmode) · [`DefragOptions`](#defragoptions) · [`DefragProgressEvent`](#defragprogressevent) · [`DefragRebuilder`](#defragrebuilder) · [`EntropyDetector`](#entropydetector) · [`FatDirStamp`](#fatdirstamp) · [`FilesystemCompressionParameter`](#filesystemcompressionparameter) · [`FilesystemCompressionProfile`](#filesystemcompressionprofile) · [`FilesystemDirectoryEntry`](#filesystemdirectoryentry) · [`FilesystemDriverBindingKind`](#filesystemdriverbindingkind) · [`FilesystemDriverCapabilities`](#filesystemdrivercapabilities) · [`FilesystemDriverCoverage`](#filesystemdrivercoverage) · [`FilesystemDriverDerivation`](#filesystemdriverderivation) · [`FilesystemDriverProfile`](#filesystemdriverprofile) · [`FilesystemDriverReadinessLayer`](#filesystemdriverreadinesslayer) · [`FilesystemDriverReadinessReport`](#filesystemdriverreadinessreport) · [`FilesystemDriverTarget`](#filesystemdrivertarget) · [`FilesystemMetadataPatch`](#filesystemmetadatapatch) · [`FilesystemMutationModel`](#filesystemmutationmodel) · [`FilesystemNodeId`](#filesystemnodeid) · [`FilesystemNodeInfo`](#filesystemnodeinfo) · [`FilesystemNodeKind`](#filesystemnodekind) · [`FilesystemOpenOptions`](#filesystemopenoptions) · [`FilesystemOptimization`](#filesystemoptimization) · [`FilesystemOptimizationAdapters`](#filesystemoptimizationadapters) · [`FilesystemOptimizationAdapters.HardLinkDeduplicator`](#filesystemoptimizationadaptershardlinkdeduplicator) · [`FilesystemOptimizationAdapters.SymbolicLinkDeduplicator`](#filesystemoptimizationadapterssymboliclinkdeduplicator) · [`FilesystemOptimizationExtensions`](#filesystemoptimizationextensions) · [`FilesystemOptimizationFeatures`](#filesystemoptimizationfeatures) · [`FilesystemOptimizationOptions`](#filesystemoptimizationoptions) · [`FilesystemSchemaPresets`](#filesystemschemapresets) · [`FilesystemSnapshotDirectoryEntry`](#filesystemsnapshotdirectoryentry) · [`FilesystemSnapshotNode`](#filesystemsnapshotnode) · [`FormatCapabilities`](#formatcapabilities) · [`FormatCategory`](#formatcategory) · [`FormatCreateOptions`](#formatcreateoptions) · [`FormatDetectionSignature`](#formatdetectionsignature) · [`FormatHeaderMatch`](#formatheadermatch) · [`FormatHealth`](#formathealth) · [`FormatHelpers`](#formathelpers) · [`FormatMethodInfo`](#formatmethodinfo) · [`FormatOptionDescriptor`](#formatoptiondescriptor) · [`FormatOptionKind`](#formatoptionkind) · [`FormatRegistry`](#formatregistry) · [`HardLinkDeduplicationSemantics`](#hardlinkdeduplicationsemantics) · [`IArchiveCreatable`](#iarchivecreatable) · [`IArchiveDefragmentable`](#iarchivedefragmentable) · [`IArchiveFormatOperations`](#iarchiveformatoperations) · [`IArchiveInMemoryExtract`](#iarchiveinmemoryextract) · [`IArchiveLayoutMap`](#iarchivelayoutmap) · [`IArchiveModifiable`](#iarchivemodifiable) · [`IArchivePurgeable`](#iarchivepurgeable) · [`IArchiveShrinkable`](#iarchiveshrinkable) · [`IArchiveWriteConstraints`](#iarchivewriteconstraints) · [`IAsyncArchiveOperations`](#iasyncarchiveoperations) · [`IAudioContainerFormat`](#iaudiocontainerformat) · [`IAudioDemuxSource`](#iaudiodemuxsource) · [`IAudioMuxTarget`](#iaudiomuxtarget) · [`IAudioPcmSource`](#iaudiopcmsource) · [`IAudioPcmTarget`](#iaudiopcmtarget) · [`IBlockDeviceFilesystemDriverProvider`](#iblockdevicefilesystemdriverprovider) · [`IBlockDeviceProvider`](#iblockdeviceprovider) · [`IBuildingBlock`](#ibuildingblock) · [`IContainerRemuxable`](#icontainerremuxable) · [`IFileInternalChunkMover`](#ifileinternalchunkmover) · [`IFileInternalLayoutMap`](#ifileinternallayoutmap) · [`IFilesystemBlockMover`](#ifilesystemblockmover) · [`IFilesystemDriverAdapter`](#ifilesystemdriveradapter) · [`IFilesystemDriverProvider`](#ifilesystemdriverprovider) · [`IFilesystemDriverReadinessProvider`](#ifilesystemdriverreadinessprovider) · [`IFilesystemExtentMap`](#ifilesystemextentmap) · [`IFilesystemFileHandle`](#ifilesystemfilehandle) · [`IFilesystemMetadataMover`](#ifilesystemmetadatamover) · [`IFilesystemPlaceable`](#ifilesystemplaceable) · [`IFilesystemScrambleable`](#ifilesystemscrambleable) · [`IFilesystemSession`](#ifilesystemsession) · [`IFilesystemTransaction`](#ifilesystemtransaction) · [`IFormatDescriptor`](#iformatdescriptor) · [`IFormatDetectionSource`](#iformatdetectionsource) · [`IFormatOptionsSchema`](#iformatoptionsschema) · [`IFormatValidator`](#iformatvalidator) · [`ILayoutOptimizable`](#ilayoutoptimizable) · [`IPartitionEditable`](#ipartitioneditable) · [`IRandomAccessBlockDevice`](#irandomaccessblockdevice) · [`IRandomAccessBlockDeviceProvider`](#irandomaccessblockdeviceprovider) · [`IRawTrackDevice`](#irawtrackdevice) · [`IRawTrackDeviceProvider`](#irawtrackdeviceprovider) · [`IStreamFormatOperations`](#istreamformatoperations) · [`ISymbolicLinkDeduplicationLayout`](#isymboliclinkdeduplicationlayout) · [`ISyntheticEntryNames`](#isyntheticentrynames) · [`IWipeEmpty`](#iwipeempty) · [`InnerFsDetector`](#innerfsdetector) · [`IssueSeverity`](#issueseverity) · [`LayoutAnalysis`](#layoutanalysis) · [`LayoutPatch`](#layoutpatch) · [`LayoutProfile`](#layoutprofile) · [`LayoutRebuildOptions`](#layoutrebuildoptions) · [`LayoutReclaim`](#layoutreclaim) · [`MagicSignature`](#magicsignature) · [`MediaProfile`](#mediaprofile) · [`MediaProfileLookup`](#mediaprofilelookup) · [`MetadataPlacementProfile`](#metadataplacementprofile) · [`MetadataPlacementRule`](#metadataplacementrule) · [`MetadataZone`](#metadatazone) · [`MethodNameParser`](#methodnameparser) · [`ModifyRebuilder`](#modifyrebuilder) · [`MutableRebuildFilesystemSession`](#mutablerebuildfilesystemsession) · [`MutableRebuildFilesystemSession.RebuildImage`](#mutablerebuildfilesystemsessionrebuildimage) · [`MutableRebuildFilesystemSession.ValidateImage`](#mutablerebuildfilesystemsessionvalidateimage) · [`PartitionBlockDevice`](#partitionblockdevice) · [`PlacementOptions`](#placementoptions) · [`PlacementZone`](#placementzone) · [`RawDiskShrinkRebuilder`](#rawdiskshrinkrebuilder) · [`RawTrackInfo`](#rawtrackinfo) · [`ReadOnlyFilesystemSnapshotSession`](#readonlyfilesystemsnapshotsession) · [`RebuildFilesystemEntry`](#rebuildfilesystementry) · [`RebuildVerb`](#rebuildverb) · [`ScrambleOptions`](#scrambleoptions) · [`SpoolingReadOnlyFileHandle`](#spoolingreadonlyfilehandle) · [`StreamBlockDevice`](#streamblockdevice) · [`SymlinkResolver`](#symlinkresolver) · [`UnusedSpaceWiper`](#unusedspacewiper) · [`ValidationIssue`](#validationissue) · [`ValidationLevel`](#validationlevel) · [`ValidationResult`](#validationresult) · [`WholeImageRebuildCommitter`](#wholeimagerebuildcommitter) #### `AlgorithmFamily` @@ -6383,6 +6383,15 @@ Implements `IEquatable`. | `InMemory` | `static ArchiveInputInfo InMemory(string archiveName, byte[] content)` | Creates an in-memory input whose content comes from `content` rather than a file on disk. | | `ReadContent` | `byte[] ReadContent()` | Returns the input's bytes: the in-memory content when present, otherwise the file at `FullPath`. Descriptors should call this instead of `File.ReadAllBytes(FullPath)` so they transparently support in-memory (temp-free) creation and conversion. | +#### `ArchiveMutationOptions` + +Per-operation options for mutating an existing archive or filesystem image. + +| Member | Signature | Summary | +| --- | --- | --- | +| `ArchiveMutationOptions` | `ArchiveMutationOptions()` | | +| `Password` | `string Password { get; init; }` | Password or passphrase required to unlock the existing container. | + #### `ArchiveShrinker` Generic shrink driver for any archive whose descriptor implements `IArchiveCreatable`. Lists the archive's entries, extracts them to memory, and re-creates a fresh archive into the output stream using the smallest canonical size that fits (from `CanonicalSizes`). Works as the default `Shrink` implementation for most R/W filesystems: the "rebuild via WORM" pattern implicitly defragments as a side effect. Format-specific shrinkers can override for efficiency. @@ -7325,7 +7334,9 @@ Implements `IArchivePurgeable`. | Member | Signature | Summary | | --- | --- | --- | | `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Adds files to an existing instance, replacing entries with the same logical path/name. Default implementation: descriptors that also implement `IArchiveFormatOperations` and `IArchiveCreatable` get a verified extract → edit → re-create implementation through `EditViaRebuild`. Formats with a cheaper native editor override it. | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs, ArchiveMutationOptions options)` | Adds or replaces files while supplying operation-scoped credentials. | | `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes the named entries from an existing instance. Passing every entry name yields an empty container/image where the format permits one. Default implementation: a verified extract → drop-named-files → re-create edit through `EditViaRebuild`. Native implementations may instead unlink/free in place and optionally wipe released storage. | +| `Remove` | `void Remove(Stream archive, string[] entryNames, ArchiveMutationOptions options)` | Removes entries while supplying operation-scoped credentials. | #### `IArchivePurgeable` @@ -8068,6 +8079,7 @@ Generic, round-trip-verified extract → re-create engine shared by maintenance | Member | Signature | Summary | | --- | --- | --- | +| `CreateScratchStream` | `static FileStream CreateScratchStream()` | A writable scratch stream not bounded by byte[] / MemoryStream size. | | `EditViaRebuild` | `static void EditViaRebuild(Stream archive, IArchiveFormatOperations ops, IArchiveCreatable creator, Action mutate)` | Rebuild-based edit used by the generic modifier. Mutation and validation happen off to the side; the original is overwritten only after a valid staged result exists. | | `PurgeViaModifier` | `static void PurgeViaModifier(Stream archive, IArchiveFormatOperations ops, IArchiveModifiable modifier)` | Transactional purge for a mutable container. The modifier operates on a staged copy and the caller's stream is replaced only after the result lists successfully with every original live entry gone. | | `RebuildInPlace` | `static void RebuildInPlace(Stream archive, IArchiveFormatOperations ops, IArchiveCreatable creator, IReadOnlyDictionary formatSpecific = null, Action onProgress = null, CancellationToken cancellationToken = null)` | Rebuilds into a scratch file, verifies it, and only then replaces the caller-supplied stream. Cancellation is honoured until commit starts; once commit begins it runs to completion so a cancellation cannot leave the original half-overwritten. | diff --git a/Compression.Registry/ArchiveMutationOptions.cs b/Compression.Registry/ArchiveMutationOptions.cs new file mode 100644 index 000000000..fb8463249 --- /dev/null +++ b/Compression.Registry/ArchiveMutationOptions.cs @@ -0,0 +1,14 @@ +namespace Compression.Registry; + +/// +/// Per-operation options for mutating an existing archive or filesystem image. +/// +/// +/// Mutation credentials belong to the operation, not to the input entries. Keeping +/// them here avoids format-specific ambient state and lets encrypted containers +/// expose the same add/replace/remove contract as unencrypted formats. +/// +public sealed class ArchiveMutationOptions { + /// Password or passphrase required to unlock the existing container. + public string? Password { get; init; } +} diff --git a/Compression.Registry/IArchiveModifiable.cs b/Compression.Registry/IArchiveModifiable.cs index 323ec77ea..c0876df2c 100644 --- a/Compression.Registry/IArchiveModifiable.cs +++ b/Compression.Registry/IArchiveModifiable.cs @@ -39,6 +39,17 @@ void Add(Stream archive, IReadOnlyList inputs) { }); } + /// Adds or replaces files while supplying operation-scoped credentials. + /// + /// The default deliberately delegates to the legacy overload so every existing + /// unencrypted modifier keeps its behavior unchanged. Password-aware formats + /// override this overload and consume . + /// + void Add(Stream archive, IReadOnlyList inputs, ArchiveMutationOptions options) { + ArgumentNullException.ThrowIfNull(options); + this.Add(archive, inputs); + } + /// /// Removes the named entries from an existing instance. Passing every entry name yields an /// empty container/image where the format permits one. @@ -60,4 +71,14 @@ void Remove(Stream archive, string[] entryNames) { } }); } + + /// Removes entries while supplying operation-scoped credentials. + /// + /// The default delegates to the legacy overload; formats that need a password to + /// locate or rewrite encrypted metadata can override this overload independently. + /// + void Remove(Stream archive, string[] entryNames, ArchiveMutationOptions options) { + ArgumentNullException.ThrowIfNull(options); + this.Remove(archive, entryNames); + } } diff --git a/Compression.Registry/IArchiveWriteConstraints.cs b/Compression.Registry/IArchiveWriteConstraints.cs index 2a94d64cb..fe436c9bb 100644 --- a/Compression.Registry/IArchiveWriteConstraints.cs +++ b/Compression.Registry/IArchiveWriteConstraints.cs @@ -4,7 +4,7 @@ namespace Compression.Registry; /// Opt-in capability: the descriptor can reject inputs that don't belong in this archive /// type. Applied by the UI (drag-drop prohibition cursor + tooltip) and the CLI (rejection /// with non-zero exit) before or -/// is called. +/// is called. /// /// Descriptors that accept anything (ZIP, TAR, …) simply don't implement this interface. /// diff --git a/Compression.Registry/RebuildVerb.cs b/Compression.Registry/RebuildVerb.cs index 42b234bd9..3ee3feec6 100644 --- a/Compression.Registry/RebuildVerb.cs +++ b/Compression.Registry/RebuildVerb.cs @@ -482,7 +482,7 @@ private static string SafeExtractPath(string root, string archiveName) { } /// A writable scratch stream not bounded by byte[] / MemoryStream size. - internal static FileStream CreateScratchStream() + public static FileStream CreateScratchStream() => new(Path.Combine(Path.GetTempPath(), "cwb_rebuild_" + Guid.NewGuid().ToString("N") + ".tmp"), FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 64 * 1024, FileOptions.DeleteOnClose); diff --git a/Compression.Tests/Cso/CsoInPlaceModifyTests.cs b/Compression.Tests/Cso/CsoInPlaceModifyTests.cs index 38ffff14b..51b326132 100644 --- a/Compression.Tests/Cso/CsoInPlaceModifyTests.cs +++ b/Compression.Tests/Cso/CsoInPlaceModifyTests.cs @@ -5,249 +5,245 @@ namespace Compression.Tests.Cso; -/// -/// Block-level in-place mutation for CSO v1 images. Two on-disk shapes are -/// exercised: (a) the new compressed block fits inside the old slot — header, -/// index, and adjacent block bytes remain byte-identical except for the -/// flag bit on the modified entry; (b) the new payload is larger than the -/// old slot — it lands at EOF and the index is patched to reflect the new -/// position while every block remains decodable through -/// . -/// [TestFixture] public class CsoInPlaceModifyTests { - private const int BlockSize = 2048; - /// - /// Produces an uncompressed payload of blocksCount × BlockSize bytes - /// where each block has a distinct fill so corruption is locally visible. - /// - private static byte[] BuildPayload(int blocksCount, byte seed) { - var data = new byte[blocksCount * BlockSize]; - for (var b = 0; b < blocksCount; ++b) - for (var i = 0; i < BlockSize; ++i) - data[b * BlockSize + i] = (byte)((seed + b * 7 + i) & 0xFF); - return data; + [TestCase(null, TestName = "WriteBlock_CsoV1")] + [TestCase("zso", TestName = "WriteBlock_Zso")] + [TestCase("cso2", TestName = "WriteBlock_CsoV2")] + [Category("RoundTrip")] + public void WriteBlock_RoundTripsModifiedAndUnmodifiedBlocks(string? variant) { + var payload = BuildPayload(3 * BlockSize, 7); + var image = CreateImage(payload, variant); + var replacement = new byte[BlockSize]; + new Random(1234).NextBytes(replacement); + + using var stream = Writable(image); + CsoInPlaceModifier.WriteBlock(stream, 1, replacement); + + var expected = payload.ToArray(); + replacement.CopyTo(expected, BlockSize); + Assert.That(ReadLogicalPayload(stream.ToArray()), Is.EqualTo(expected)); } - /// Builds a fresh CSO image around the supplied uncompressed payload. - private static byte[] BuildImage(byte[] payload) => CsoWriter.Build(payload); - - /// Reads the in-place index entries (without the flag bit) of a CSO stream. - private static (long[] Offsets, bool[] StoredFlags) ReadIndex(byte[] image) { - var blockSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(16, 4)); - var uncompressed = (long)BinaryPrimitives.ReadUInt64LittleEndian(image.AsSpan(8, 8)); - var blockCount = (int)((uncompressed + blockSize - 1) / blockSize); - var indexCount = blockCount + 1; - var offsets = new long[indexCount]; - var flags = new bool[indexCount]; - for (var i = 0; i < indexCount; ++i) { - var raw = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(24 + i * 4, 4)); - flags[i] = (raw & 0x8000_0000u) != 0; - offsets[i] = raw & 0x7FFF_FFFFu; - } - return (offsets, flags); - } - - [Test, Category("HappyPath")] - public void WriteBlock_FitsInPlace_LeavesHeaderAndOtherBlocksByteIdentical() { - var payload = BuildPayload(4, seed: 1); - var image = BuildImage(payload); - - // Choose a block to overwrite with HIGHLY-compressible content (zeros) - // so the new compressed bytes are guaranteed smaller than the old slot. - var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - var (oldOffsets, _) = ReadIndex(image); - var modifiedBlock = 1; - var oldSize = oldOffsets[modifiedBlock + 1] - oldOffsets[modifiedBlock]; - Assert.That(oldSize, Is.GreaterThan(0), "the chosen block must have non-zero on-disk size"); - - var zeros = new byte[BlockSize]; - CsoInPlaceModifier.WriteBlock(ms, modifiedBlock, zeros); - - var after = ms.ToArray(); - - // Stream length unchanged. - Assert.That(after.Length, Is.EqualTo(image.Length), - "in-place WriteBlock with a smaller payload must not grow the stream"); - - // Header bytes byte-identical (offsets 0..23). - Assert.That(after.AsSpan(0, 24).SequenceEqual(image.AsSpan(0, 24)), Is.True, - "header must be byte-identical after in-place WriteBlock"); - - // Index entries (offsets) unchanged for the in-place case; flag may flip. - var (newOffsets, _) = ReadIndex(after); - for (var i = 0; i < newOffsets.Length; ++i) - Assert.That(newOffsets[i], Is.EqualTo(oldOffsets[i]), - $"in-place modify must not move block {i} on disk"); - - // Every other block's bytes are byte-identical. - for (var i = 0; i < oldOffsets.Length - 1; ++i) { - if (i == modifiedBlock) continue; - var start = (int)oldOffsets[i]; - var end = (int)oldOffsets[i + 1]; - Assert.That(after.AsSpan(start, end - start).SequenceEqual(image.AsSpan(start, end - start)), - Is.True, $"block {i} bytes must not move when block {modifiedBlock} is rewritten in place"); - } + [Test, Category("Compatibility")] + public void WriteBlock_NonZeroIndexShift_CanonicalizesAndPreservesContent() { + var payload = BuildPayload(2 * BlockSize, 29); + var image = BuildShiftedStoredImage(payload); + var replacement = Enumerable.Repeat((byte)0xA5, BlockSize).ToArray(); + + using var stream = Writable(image); + CsoInPlaceModifier.WriteBlock(stream, 0, replacement); + var after = stream.ToArray(); + + Assert.That(after[21], Is.Zero, "transactional repack should canonicalize index_shift to zero"); + var expected = payload.ToArray(); + replacement.CopyTo(expected, 0); + Assert.That(ReadLogicalPayload(after), Is.EqualTo(expected)); } - [Test, Category("HappyPath")] - public void WriteBlock_FitsInPlace_RoundTripsModifiedAndUnmodifiedBlocks() { - var payload = BuildPayload(3, seed: 42); - var image = BuildImage(payload); - - var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - - var zeros = new byte[BlockSize]; - CsoInPlaceModifier.WriteBlock(ms, 1, zeros); - - // Extract each block and assert the modified one is zeros, others are intact. - var desc = new CsoFormatDescriptor(); - var tmp = Path.Combine(Path.GetTempPath(), "cso_inplace_test_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(tmp); - try { - ms.Position = 0; - desc.Extract(ms, tmp, null, ["block_00000.bin", "block_00001.bin", "block_00002.bin"]); - - var b0 = File.ReadAllBytes(Path.Combine(tmp, "blocks/block_00000.bin")); - var b1 = File.ReadAllBytes(Path.Combine(tmp, "blocks/block_00001.bin")); - var b2 = File.ReadAllBytes(Path.Combine(tmp, "blocks/block_00002.bin")); - - // List reports each block's compressed size; descriptor's Extract writes - // the on-disk (still-compressed) bytes. We only assert structural - // properties — the modified entry must round-trip a decompressed-zero - // slab when inflated, which the FULL.cso path covers. - Assert.That(b0.Length, Is.GreaterThan(0)); - Assert.That(b1.Length, Is.GreaterThan(0), "modified block on disk has compressed-zeros payload"); - Assert.That(b2.Length, Is.GreaterThan(0)); - } finally { - Directory.Delete(tmp, recursive: true); - } + [Test, Category("RoundTrip")] + public void Descriptor_Add_ReplacesSeveralBlocksInOneOperation() { + var payload = BuildPayload(4 * BlockSize, 3); + var image = CreateImage(payload, "zso"); + var block0 = Enumerable.Repeat((byte)0x11, BlockSize).ToArray(); + var block3 = Enumerable.Repeat((byte)0xCC, BlockSize).ToArray(); + + using var stream = Writable(image); + IArchiveModifiable modifier = new CsoFormatDescriptor(); + modifier.Add(stream, [ + ArchiveInputInfo.InMemory("blocks/block_00000.bin", block0), + ArchiveInputInfo.InMemory("blocks/block_00003.bin", block3), + ]); + + var expected = payload.ToArray(); + block0.CopyTo(expected, 0); + block3.CopyTo(expected, 3 * BlockSize); + Assert.That(ReadLogicalPayload(stream.ToArray()), Is.EqualTo(expected)); } - [Test, Category("HappyPath")] - public void WriteBlock_LargerThanSlot_HeaderPrefixUnchanged_OtherBlocksDecodable() { - // Build payload where blocks are highly compressible so the writer emits - // small compressed bytes, then over-write block 0 with random data whose - // DEFLATE output is larger than the original tiny slot — forces the - // append-at-EOF branch. - var payload = new byte[3 * BlockSize]; // zeros → DEFLATE compresses tiny - var image = BuildImage(payload); - var (oldOffsets, _) = ReadIndex(image); - var oldBlock0Size = oldOffsets[1] - oldOffsets[0]; + [Test, Category("ErrorHandling")] + public void WriteBlock_BlockIndexOutOfRange_Throws() { + using var stream = Writable(CreateImage(BuildPayload(2 * BlockSize, 9))); + Assert.That( + () => CsoInPlaceModifier.WriteBlock(stream, 99, new byte[BlockSize]), + Throws.InstanceOf()); + } - // Random bytes don't compress; their DEFLATE output will be > oldBlock0Size. - var random = new byte[BlockSize]; - new Random(1).NextBytes(random); + [Test, Category("ErrorHandling")] + public void WriteBlock_WrongPayloadSize_Throws() { + using var stream = Writable(CreateImage(BuildPayload(2 * BlockSize, 9))); + Assert.That( + () => CsoInPlaceModifier.WriteBlock(stream, 0, new byte[BlockSize - 1]), + Throws.InstanceOf()); + } - var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - var oldLen = (int)ms.Length; - CsoInPlaceModifier.WriteBlock(ms, 0, random); + [Test, Category("Capabilities")] + public void Descriptor_AdvertisesReadWriteAndMaintenanceInterfaces() { + var descriptor = new CsoFormatDescriptor(); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + } - var after = ms.ToArray(); + [Test, Category("Maintenance")] + public void Defragment_DropsUnindexedTailAndPreservesLogicalIso() { + var payload = BuildPayload(3 * BlockSize, 41); + var image = CreateImage(payload, "cso2"); + using var stream = Writable(AppendGarbage(image, 193)); + var oldLength = stream.Length; - // Stream grew because the new payload didn't fit in place. - Assert.That(after.Length, Is.GreaterThan(oldLen), - "grow-then-append branch must extend the stream"); + ((IArchiveDefragmentable)new CsoFormatDescriptor()).Defragment(stream); - // Header pre-fix unchanged (offsets 0..23 carry magic + uncompressed_size + - // block_size + version + align; they are NOT touched by in-place writes). - Assert.That(after.AsSpan(0, 24).SequenceEqual(image.AsSpan(0, 24)), Is.True, - "the 24-byte header must remain byte-identical after a grow-append modify"); + Assert.That(stream.Length, Is.LessThan(oldLength)); + Assert.That(ReadLogicalPayload(stream.ToArray()), Is.EqualTo(payload)); + Assert.That(stream.ToArray()[21], Is.Zero); + } - // Index moved: block 0 now points past the old tail. - var (newOffsets, _) = ReadIndex(after); - Assert.That(newOffsets[0], Is.GreaterThanOrEqualTo(oldLen), - "moved block 0 must land past the original EOF"); + [Test, Category("Maintenance")] + public void WipeUnusedSpace_ZeroesCanonicalizedTailWithoutChangingLength() { + var payload = BuildPayload(2 * BlockSize, 55); + var core = CreateImage(payload, "zso"); + using var stream = Writable(AppendGarbage(core, 257)); + var originalLength = stream.Length; - // Every block still decodable via FULL.cso round-trip (header + index + bodies - // describe a coherent CSO container). - var desc = new CsoFormatDescriptor(); - ms.Position = 0; - var entries = desc.List(ms, null); - Assert.That(entries.Count(e => e.Name.StartsWith("blocks/block_")), Is.EqualTo(3), - "post-grow image must still surface 3 synthetic block entries"); + var wiped = ((IWipeEmpty)new CsoFormatDescriptor()).WipeUnusedSpace(stream); + + Assert.That(wiped, Is.EqualTo(257)); + Assert.That(stream.Length, Is.EqualTo(originalLength)); + Assert.That(stream.ToArray().AsSpan(core.Length).ToArray(), Is.All.Zero); + Assert.That(ReadLogicalPayload(stream.ToArray()), Is.EqualTo(payload)); } - [Test, Category("ErrorHandling")] - public void WriteBlock_BlockIndexOutOfRange_Throws() { - var image = BuildImage(BuildPayload(2, seed: 9)); - using var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - Assert.That( - () => CsoInPlaceModifier.WriteBlock(ms, 99, new byte[BlockSize]), - Throws.InstanceOf(), - "out-of-range block index must throw, not corrupt the image"); + [Test, Category("Maintenance")] + public void Shrink_DropsUnindexedTailAndNeverGrows() { + var payload = BuildPayload(2 * BlockSize, 71); + var sourceBytes = AppendGarbage(CreateImage(payload), 311); + using var source = new MemoryStream(sourceBytes, writable: false); + using var target = new MemoryStream(); + + ((IArchiveShrinkable)new CsoFormatDescriptor()).Shrink(source, target); + + Assert.That(target.Length, Is.LessThan(sourceBytes.Length)); + Assert.That(ReadLogicalPayload(target.ToArray()), Is.EqualTo(payload)); } - [Test, Category("ErrorHandling")] - public void WriteBlock_WrongPayloadSize_Throws() { - var image = BuildImage(BuildPayload(2, seed: 9)); - using var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - Assert.That( - () => CsoInPlaceModifier.WriteBlock(ms, 0, new byte[BlockSize - 1]), - Throws.InstanceOf(), - "payload must be exactly block_size bytes"); + [Test, Category("Maintenance")] + public void RebuildStreaming_ReblocksWithoutChangingLogicalIso() { + var payload = BuildPayload(5 * BlockSize, 87); + var sourceBytes = CreateImage(payload, "cso2"); + using var source = new MemoryStream(sourceBytes, writable: false); + using var target = new MemoryStream(); + + ((ILayoutOptimizable)new CsoFormatDescriptor()).RebuildStreaming( + source, target, new LayoutRebuildOptions { UnitSize = 4096 }); + + var rebuilt = target.ToArray(); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(rebuilt.AsSpan(16, 4)), Is.EqualTo(4096)); + Assert.That(ReadLogicalPayload(rebuilt), Is.EqualTo(payload)); } - [Test, Category("RoundTrip")] - public void MutateThenExtract_RoundTrip() { - var payload = BuildPayload(2, seed: 7); - var image = BuildImage(payload); - - using var ms = new MemoryStream(); - ms.Write(image, 0, image.Length); - - // Mutate block 0 to all-zeros via the descriptor's IArchiveModifiable surface. - IArchiveModifiable mod = new CsoFormatDescriptor(); - mod.Add(ms, [ArchiveInputInfo.InMemory("blocks/block_00000.bin", new byte[BlockSize])]); - - // Extract and verify FULL.cso re-reads the mutated image without error. - var desc = new CsoFormatDescriptor(); - var tmp = Path.Combine(Path.GetTempPath(), "cso_mutate_extract_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(tmp); - try { - ms.Position = 0; - desc.Extract(ms, tmp, null, ["FULL.cso", "metadata.ini"]); - Assert.That(File.Exists(Path.Combine(tmp, "FULL.cso")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "metadata.ini")), Is.True); - } finally { - Directory.Delete(tmp, recursive: true); + [Test, Category("Maintenance")] + public void LayoutMap_DeclaresOnlyTrailingUnindexedBytesFree() { + var image = AppendGarbage(CreateImage(BuildPayload(BlockSize, 101)), 64); + using var stream = new MemoryStream(image, writable: false); + + var layout = ((IArchiveLayoutMap)new CsoFormatDescriptor()).EnumerateLayout(stream).ToArray(); + + Assert.That(layout[0].Kind, Is.EqualTo(DefragBlockKind.MetadataReserved)); + Assert.That(layout.Any(e => e.Kind == DefragBlockKind.Used), Is.True); + var free = layout.Single(e => e.Kind == DefragBlockKind.Free); + Assert.That(free.Length, Is.EqualTo(64)); + Assert.That(free.Offset + free.Length, Is.EqualTo(image.Length)); + } + + [Test, Category("Maintenance")] + public void Purge_LeavesValidEmptyContainerOfSameVariant() { + using var stream = Writable(CreateImage(BuildPayload(2 * BlockSize, 111), "zso")); + + ((IArchivePurgeable)new CsoFormatDescriptor()).Purge(stream); + + var bytes = stream.ToArray(); + Assert.That(System.Text.Encoding.ASCII.GetString(bytes, 0, 4), Is.EqualTo("ZISO")); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(bytes.AsSpan(8, 8)), Is.Zero); + stream.Position = 0; + var entries = new CsoFormatDescriptor().List(stream, null); + Assert.That(entries.Count(e => e.Name.StartsWith("blocks/block_", StringComparison.Ordinal)), Is.Zero); + } + + private static MemoryStream Writable(byte[] bytes) { + var stream = new MemoryStream(); + stream.Write(bytes); + stream.Position = 0; + return stream; + } + + private static byte[] CreateImage(byte[] payload, string? variant = null) { + var creator = (IArchiveCreatable)new CsoFormatDescriptor(); + using var stream = new MemoryStream(); + creator.Create(stream, [ArchiveInputInfo.InMemory("disc.iso", payload)], new FormatCreateOptions(variant)); + return stream.ToArray(); + } + + private static byte[] ReadLogicalPayload(byte[] image) { + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image, writable: false); + var names = descriptor.List(stream, null) + .Where(e => !e.IsDirectory && e.Name.StartsWith("blocks/block_", StringComparison.Ordinal)) + .OrderBy(e => e.Index) + .Select(e => e.Name) + .ToArray(); + using var result = new MemoryStream(); + foreach (var name in names) { + stream.Position = 0; + result.Write(((IArchiveFormatOperations)descriptor).ExtractEntryToMemory(stream, name, null)); } + return result.ToArray(); } - [Test, Category("HappyPath")] - public void Descriptor_CapabilitiesAdvertiseCreateAndModify() { - var d = new CsoFormatDescriptor(); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True, - "Cso descriptor must advertise CanCreate"); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True, - "Cso descriptor must advertise CanModify"); - Assert.That(d, Is.InstanceOf()); - Assert.That(d, Is.InstanceOf()); + private static byte[] AppendGarbage(byte[] image, int count) { + var result = new byte[image.Length + count]; + image.CopyTo(result, 0); + result.AsSpan(image.Length).Fill(0xD7); + return result; } - [Test, Category("RoundTrip")] - public void Create_FromInputs_ProducesValidCso() { - // Build a 4 KB uncompressed payload by feeding two 2 KB inputs in order. - var first = new byte[BlockSize]; Array.Fill(first, (byte)0xAA); - var second = new byte[BlockSize]; Array.Fill(second, (byte)0x55); - - var d = (IArchiveCreatable)new CsoFormatDescriptor(); - using var ms = new MemoryStream(); - d.Create(ms, - [ArchiveInputInfo.InMemory("part1.bin", first), - ArchiveInputInfo.InMemory("part2.bin", second)], - new FormatCreateOptions()); - - ms.Position = 0; - var entries = new CsoFormatDescriptor().List(ms, null); - Assert.That(entries.Count(e => e.Name.StartsWith("blocks/block_")), Is.EqualTo(2)); - Assert.That(entries.First(e => e.Name == "FULL.cso").OriginalSize, Is.EqualTo(ms.Length)); + private static byte[] BuildShiftedStoredImage(byte[] payload) { + Assert.That(payload.Length, Is.EqualTo(2 * BlockSize)); + const byte shift = 1; + const int indexEnd = 24 + 3 * 4; + const int dataStart = 40; // even and deliberately padded beyond the index table. + var result = new byte[dataStart + payload.Length]; + "CISO"u8.CopyTo(result); + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(4, 4), 24); + BinaryPrimitives.WriteUInt64LittleEndian(result.AsSpan(8, 8), checked((ulong)payload.Length)); + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(16, 4), BlockSize); + result[20] = 1; + result[21] = shift; + + var first = checked((uint)(dataStart >> shift)) | 0x8000_0000u; + var second = checked((uint)((dataStart + BlockSize) >> shift)) | 0x8000_0000u; + var end = checked((uint)((dataStart + payload.Length) >> shift)); + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(24, 4), first); + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(28, 4), second); + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(32, 4), end); + Assert.That(indexEnd, Is.EqualTo(36)); + payload.CopyTo(result, dataStart); + return result; + } + + private static byte[] BuildPayload(int length, byte seed) { + var data = new byte[length]; + for (var i = 0; i < data.Length; ++i) + data[i] = (byte)(seed + i * 13 + i / 97); + return data; } } diff --git a/Compression.Tests/Cso/CsoTests.cs b/Compression.Tests/Cso/CsoTests.cs index 5ae960eec..60c83036f 100644 --- a/Compression.Tests/Cso/CsoTests.cs +++ b/Compression.Tests/Cso/CsoTests.cs @@ -1,126 +1,111 @@ using System.Buffers.Binary; +using Compression.Registry; +using FileFormat.Cso; namespace Compression.Tests.Cso; [TestFixture] public class CsoTests { - - // Build a minimal synthetic CSO (or ZSO): 24-byte header, N+1 index entries, then fake block data. - // uncompressed_size = 4096, block_size = 2048 => 2 blocks, 3 index entries. - private static byte[] BuildSyntheticImage(bool isZso, byte[] block0, byte[] block1, bool block0Stored = false, bool block1Stored = false) { - using var ms = new MemoryStream(); - ms.Write(isZso ? "ZISO"u8 : "CISO"u8); - Span buf = stackalloc byte[8]; - // header_size = 0x18 (24) - BinaryPrimitives.WriteUInt32LittleEndian(buf[..4], 0x18u); - ms.Write(buf[..4]); - // uncompressed_size = 4096 - BinaryPrimitives.WriteUInt64LittleEndian(buf, 4096UL); - ms.Write(buf); - // block_size = 2048 - BinaryPrimitives.WriteUInt32LittleEndian(buf[..4], 2048u); - ms.Write(buf[..4]); - // version, align, reserved[2] - ms.WriteByte(1); - ms.WriteByte(0); - ms.WriteByte(0); - ms.WriteByte(0); - - // Index table: 3 entries. Offsets point to byte positions in the file. - var indexStart = (uint)ms.Position; - var indexSize = 3u * 4u; // 12 bytes - var block0Offset = indexStart + indexSize; // immediately after index - var block1Offset = block0Offset + (uint)block0.Length; - var endOffset = block1Offset + (uint)block1.Length; - - uint entry0 = block0Offset | (block0Stored ? 0x8000_0000u : 0u); - uint entry1 = block1Offset | (block1Stored ? 0x8000_0000u : 0u); - uint entry2 = endOffset; - - BinaryPrimitives.WriteUInt32LittleEndian(buf[..4], entry0); - ms.Write(buf[..4]); - BinaryPrimitives.WriteUInt32LittleEndian(buf[..4], entry1); - ms.Write(buf[..4]); - BinaryPrimitives.WriteUInt32LittleEndian(buf[..4], entry2); - ms.Write(buf[..4]); - - ms.Write(block0); - ms.Write(block1); - return ms.ToArray(); - } + private const int BlockSize = 2048; [Test, Category("HappyPath")] public void Descriptor_Properties() { - var d = new FileFormat.Cso.CsoFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("Cso")); - Assert.That(d.Extensions, Contains.Item(".cso")); - Assert.That(d.Extensions, Contains.Item(".ziso")); - Assert.That(d.Category, Is.EqualTo(Compression.Registry.FormatCategory.Archive)); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(2)); + var descriptor = new CsoFormatDescriptor(); + Assert.That(descriptor.Id, Is.EqualTo("Cso")); + Assert.That(descriptor.Extensions, Does.Contain(".cso")); + Assert.That(descriptor.Extensions, Does.Contain(".ziso")); + Assert.That(descriptor.Extensions, Does.Contain(".zso")); + Assert.That(descriptor.Category, Is.EqualTo(FormatCategory.Archive)); + Assert.That(descriptor.MagicSignatures, Has.Count.EqualTo(2)); } - [Test, Category("HappyPath")] - public void List_SurfacesBlocksAndIndex() { - var block0 = new byte[] { 0x10, 0x20, 0x30 }; - var block1 = new byte[] { 0xAA, 0xBB }; - var img = BuildSyntheticImage(isZso: false, block0, block1, block0Stored: false, block1Stored: true); - - var desc = new FileFormat.Cso.CsoFormatDescriptor(); - using var ms = new MemoryStream(img); - var entries = desc.List(ms, null); - - // FULL.cso, metadata.ini, index.bin, blocks (dir), block_00000.bin, block_00001.bin = 6 entries - Assert.That(entries, Has.Count.EqualTo(6)); - Assert.That(entries[0].Name, Is.EqualTo("FULL.cso")); - Assert.That(entries[1].Name, Is.EqualTo("metadata.ini")); - Assert.That(entries[2].Name, Is.EqualTo("index.bin")); - Assert.That(entries[2].OriginalSize, Is.EqualTo(12)); - Assert.That(entries[3].Name, Is.EqualTo("blocks")); - Assert.That(entries[3].IsDirectory, Is.True); - Assert.That(entries[4].Name, Is.EqualTo("blocks/block_00000.bin")); - Assert.That(entries[4].OriginalSize, Is.EqualTo(block0.Length)); - Assert.That(entries[4].Method, Is.EqualTo("Deflate")); - Assert.That(entries[5].Name, Is.EqualTo("blocks/block_00001.bin")); - Assert.That(entries[5].OriginalSize, Is.EqualTo(block1.Length)); - Assert.That(entries[5].Method, Is.EqualTo("Stored")); + [TestCase(null, "CISO", 1, TestName = "CreateRoundTrip_CsoV1")] + [TestCase("zso", "ZISO", 1, TestName = "CreateRoundTrip_Zso")] + [TestCase("cso2", "CISO", 2, TestName = "CreateRoundTrip_CsoV2")] + [Category("RoundTrip")] + public void CreateAndLogicalBlockExtraction_RoundTrips(string? method, string magic, byte version) { + var payload = BuildPayload(BlockSize * 3 + 317); + var image = CreateImage(payload, method); + + Assert.That(System.Text.Encoding.ASCII.GetString(image, 0, 4), Is.EqualTo(magic)); + Assert.That(image[20], Is.EqualTo(version)); + Assert.That(ReadLogicalPayload(image), Is.EqualTo(payload)); + } + + [Test, Category("Interoperability")] + public void Zso_DecodesLibLz4KnownBlock() { + // liblz4 raw-block oracle for 2048 zero bytes (no frame header / no stored size prefix). + var encoded = Convert.FromHexString("1F000100FFFFFFFFFFFFFFEE500000000000"); + var image = BuildSingleBlockImage("ZISO", 1, encoded, methodFlag: false); + + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image); + var block = ((IArchiveFormatOperations)descriptor).ExtractEntryToMemory(stream, "blocks/block_00000.bin", null); + Assert.That(block, Is.EqualTo(new byte[BlockSize])); + + stream.Position = 0; + Assert.That(descriptor.List(stream, null).Single(e => e.Name == "blocks/block_00000.bin").Method, + Is.EqualTo("LZ4")); + } + + [Test, Category("Interoperability")] + public void CsoV2_DecodesLibLz4KnownBlock() { + var encoded = Convert.FromHexString("1F000100FFFFFFFFFFFFFFEE500000000000"); + var image = BuildSingleBlockImage("CISO", 2, encoded, methodFlag: true); + + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image); + var block = ((IArchiveFormatOperations)descriptor).ExtractEntryToMemory(stream, "blocks/block_00000.bin", null); + Assert.That(block, Is.EqualTo(new byte[BlockSize])); + + stream.Position = 0; + Assert.That(descriptor.List(stream, null).Single(e => e.Name == "blocks/block_00000.bin").Method, + Is.EqualTo("LZ4")); + } + + [Test, Category("Compatibility")] + public void CsoVersionZero_IsAcceptedAsV1() { + var payload = BuildPayload(BlockSize); + var image = CreateImage(payload); + image[20] = 0; + + Assert.That(ReadLogicalPayload(image), Is.EqualTo(payload)); } [Test, Category("HappyPath")] - public void List_ZsoReportsLz4Method() { - var img = BuildSyntheticImage(isZso: true, new byte[] { 1, 2 }, new byte[] { 3, 4 }); - var desc = new FileFormat.Cso.CsoFormatDescriptor(); - using var ms = new MemoryStream(img); - var entries = desc.List(ms, null); - Assert.That(entries[0].Name, Is.EqualTo("FULL.ziso")); - Assert.That(entries[4].Method, Is.EqualTo("LZ4")); + public void List_ReportsLogicalAndPhysicalBlockSizes() { + var payload = new byte[BlockSize + 137]; + var image = CreateImage(payload, "zso"); + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image); + + var entries = descriptor.List(stream, null); + var first = entries.Single(e => e.Name == "blocks/block_00000.bin"); + var last = entries.Single(e => e.Name == "blocks/block_00001.bin"); + Assert.That(first.OriginalSize, Is.EqualTo(BlockSize)); + Assert.That(last.OriginalSize, Is.EqualTo(137)); + Assert.That(first.CompressedSize, Is.LessThan(BlockSize)); + Assert.That(first.Method, Is.EqualTo("LZ4")); } [Test, Category("HappyPath")] - public void Extract_WritesBlocksAndMetadata() { - var block0 = new byte[] { 0xDE, 0xAD, 0xBE, 0xEF }; - var block1 = new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 }; - var img = BuildSyntheticImage(isZso: false, block0, block1); - - var desc = new FileFormat.Cso.CsoFormatDescriptor(); - var tmp = Path.Combine(Path.GetTempPath(), "cso_test_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(tmp); + public void Extract_WritesMetadataAndLogicalBlocks() { + var payload = BuildPayload(BlockSize * 2); + var image = CreateImage(payload, "cso2"); + var descriptor = new CsoFormatDescriptor(); + var directory = Path.Combine(Path.GetTempPath(), "cso_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); try { - using var ms = new MemoryStream(img); - desc.Extract(ms, tmp, null, null); - - Assert.That(File.Exists(Path.Combine(tmp, "FULL.cso")), Is.True); - Assert.That(File.ReadAllBytes(Path.Combine(tmp, "FULL.cso")), Is.EqualTo(img)); - Assert.That(File.Exists(Path.Combine(tmp, "metadata.ini")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "index.bin")), Is.True); - Assert.That(File.ReadAllBytes(Path.Combine(tmp, "blocks/block_00000.bin")), Is.EqualTo(block0)); - Assert.That(File.ReadAllBytes(Path.Combine(tmp, "blocks/block_00001.bin")), Is.EqualTo(block1)); - var meta = File.ReadAllText(Path.Combine(tmp, "metadata.ini")); - Assert.That(meta, Does.Contain("uncompressed_size=4096")); - Assert.That(meta, Does.Contain("block_size=2048")); - Assert.That(meta, Does.Contain("block_count=2")); - Assert.That(meta, Does.Contain("is_zso=0")); + using var stream = new MemoryStream(image); + descriptor.Extract(stream, directory, null, ["metadata.ini", "block_00001.bin"]); + + var metadata = File.ReadAllText(Path.Combine(directory, "metadata.ini")); + Assert.That(metadata, Does.Contain("variant=cso2")); + Assert.That(metadata, Does.Contain("block_size=2048")); + Assert.That(File.ReadAllBytes(Path.Combine(directory, "blocks/block_00001.bin")), + Is.EqualTo(payload.AsSpan(BlockSize, BlockSize).ToArray())); } finally { - Directory.Delete(tmp, recursive: true); + Directory.Delete(directory, recursive: true); } } @@ -128,8 +113,76 @@ public void Extract_WritesBlocksAndMetadata() { public void List_RejectsInvalidMagic() { var bogus = new byte[64]; bogus[0] = (byte)'X'; - var desc = new FileFormat.Cso.CsoFormatDescriptor(); - using var ms = new MemoryStream(bogus); - Assert.That(() => desc.List(ms, null), Throws.InstanceOf()); + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(bogus); + Assert.That(() => descriptor.List(stream, null), Throws.InstanceOf()); + } + + [Test, Category("ErrorHandling")] + public void List_RejectsNonMonotonicIndex() { + var image = CreateImage(new byte[BlockSize * 2]); + var first = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(24, 4)); + BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(28, 4), first - 1); + + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image); + Assert.That(() => descriptor.List(stream, null), Throws.InstanceOf()); + } + + private static byte[] CreateImage(byte[] payload, string? method = null, int blockSize = BlockSize) { + var creator = (IArchiveCreatable)new CsoFormatDescriptor(); + using var stream = new MemoryStream(); + var options = new FormatCreateOptions(method); + options.FormatSpecific["BlockSize"] = blockSize.ToString(System.Globalization.CultureInfo.InvariantCulture); + creator.Create(stream, [ArchiveInputInfo.InMemory("disc.iso", payload)], options); + return stream.ToArray(); + } + + private static byte[] ReadLogicalPayload(byte[] image) { + var descriptor = new CsoFormatDescriptor(); + using var stream = new MemoryStream(image, writable: false); + var names = descriptor.List(stream, null) + .Where(e => !e.IsDirectory && e.Name.StartsWith("blocks/block_", StringComparison.Ordinal)) + .OrderBy(e => e.Index) + .Select(e => e.Name) + .ToArray(); + using var output = new MemoryStream(); + foreach (var name in names) { + stream.Position = 0; + output.Write(((IArchiveFormatOperations)descriptor).ExtractEntryToMemory(stream, name, null)); + } + return output.ToArray(); + } + + private static byte[] BuildSingleBlockImage(string magic, byte version, byte[] encoded, bool methodFlag) { + using var stream = new MemoryStream(); + stream.Write(System.Text.Encoding.ASCII.GetBytes(magic)); + Span word = stackalloc byte[8]; + BinaryPrimitives.WriteUInt32LittleEndian(word[..4], 24); + stream.Write(word[..4]); + BinaryPrimitives.WriteUInt64LittleEndian(word, BlockSize); + stream.Write(word); + BinaryPrimitives.WriteUInt32LittleEndian(word[..4], BlockSize); + stream.Write(word[..4]); + stream.WriteByte(version); + stream.WriteByte(0); + stream.WriteByte(0); + stream.WriteByte(0); + + const uint dataOffset = 32; + var first = dataOffset | (methodFlag ? 0x8000_0000u : 0u); + BinaryPrimitives.WriteUInt32LittleEndian(word[..4], first); + stream.Write(word[..4]); + BinaryPrimitives.WriteUInt32LittleEndian(word[..4], dataOffset + checked((uint)encoded.Length)); + stream.Write(word[..4]); + stream.Write(encoded); + return stream.ToArray(); + } + + private static byte[] BuildPayload(int length) { + var payload = new byte[length]; + for (var i = 0; i < payload.Length; ++i) + payload[i] = (byte)((i * 17 + i / 31) & 0xFF); + return payload; } } diff --git a/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs b/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs index 28a774719..88c66693b 100644 --- a/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs +++ b/Compression.Tests/Ecryptfs/EcryptfsDetectionTests.cs @@ -1,55 +1,78 @@ -using System.Buffers.Binary; using Compression.Registry; +using FileSystem.Ecryptfs; namespace Compression.Tests.Ecryptfs; [TestFixture] public class EcryptfsDetectionTests { - // Build a minimal eCryptfs file: marker 0x3C81B7F5 + 8-byte BE decrypted size + 4-byte flags + 4-byte extent-size, - // padded out to one extent so the reader can carve the opaque ciphertext blob. - private static byte[] BuildMinimalFile(ulong decryptedSize = 4096, uint flags = 0, uint extentSize = 4096, int cipherLen = 4096) { - var image = new byte[Math.Max((int)extentSize, 64) + cipherLen]; - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(0, 4), 0x3C81B7F5u); - BinaryPrimitives.WriteUInt64BigEndian(image.AsSpan(4, 8), decryptedSize); - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(12, 4), flags); - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(16, 4), extentSize); - for (var i = 0; i < cipherLen; i++) image[Math.Max((int)extentSize, 64) + i] = (byte)((i * 7) ^ 0xA5); - return image; + private static byte[] CreateImage(byte[] content, string password = "foo", string? encryptionMethod = null) { + var descriptor = new EcryptfsFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create( + image, + [ArchiveInputInfo.InMemory("payload.bin", content)], + new FormatCreateOptions { Password = password, EncryptionMethod = encryptionMethod }); + return image.ToArray(); } [Test, Category("HappyPath")] - public void Descriptor_Properties_AndMagic() { - var d = new FileSystem.Ecryptfs.EcryptfsFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("Ecryptfs")); - Assert.That(d.DisplayName, Is.EqualTo("eCryptfs")); - Assert.That(d.Extensions, Does.Contain(".ecryptfs")); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(1)); - Assert.That(d.MagicSignatures[0].Offset, Is.EqualTo(0)); - Assert.That(d.MagicSignatures[0].Bytes, Is.EqualTo(new byte[] { 0x3C, 0x81, 0xB7, 0xF5 })); - Assert.That(d, Is.Not.InstanceOf()); + public void Descriptor_AdvertisesPasswordAwareRw_WithoutFakeMagic() { + var descriptor = new EcryptfsFormatDescriptor(); + Assert.That(descriptor.Id, Is.EqualTo("Ecryptfs")); + Assert.That(descriptor.DisplayName, Is.EqualTo("eCryptfs")); + Assert.That(descriptor.Extensions, Does.Contain(".ecryptfs")); + Assert.That(descriptor.MagicSignatures, Is.Empty, + "The eCryptfs marker is a relation between two random words, not fixed bytes."); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.SupportsPassword), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); } [Test, Category("HappyPath")] - public void Read_MinimalFile_SurfacesHeader() { - using var ms = new MemoryStream(BuildMinimalFile(decryptedSize: 12345, flags: 0x10, extentSize: 4096, cipherLen: 8192)); - var r = new FileSystem.Ecryptfs.EcryptfsReader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.Marker, Is.EqualTo(0x3C81B7F5u)); - Assert.That(r.DecryptedSize, Is.EqualTo(12345ul)); - Assert.That(r.Flags, Is.EqualTo(0x10u)); - Assert.That(r.ExtentSize, Is.EqualTo(4096u)); - - var names = r.Entries.Select(e => e.Name).ToHashSet(); - Assert.That(names, Does.Contain("FULL.ecryptfs")); - Assert.That(names, Does.Contain("metadata.ini")); - Assert.That(names, Does.Contain("ciphertext.bin")); + public void Read_CreatedLowerFile_SurfacesKernelHeaderGeometry() { + var content = Enumerable.Range(0, 12345).Select(i => (byte)(i * 37)).ToArray(); + using var image = new MemoryStream(CreateImage(content), writable: false); + using var reader = new EcryptfsReader(image); + + Assert.That(reader.Marker, Is.EqualTo(0x3C81B7F5u)); + Assert.That(reader.DecryptedSize, Is.EqualTo(12345ul)); + Assert.That(reader.FileVersion, Is.EqualTo(3)); + Assert.That(reader.Flags & 0x00000002u, Is.Not.Zero); + Assert.That(reader.ExtentSize, Is.EqualTo(4096u)); + Assert.That(reader.HeaderExtentCount, Is.EqualTo(2)); + Assert.That(reader.MetadataSize, Is.EqualTo(8192)); + Assert.That(reader.CanonicalLength, Is.EqualTo(8192 + 4 * 4096)); + Assert.That(reader.CipherDescription, Is.EqualTo("AES-128-CBC")); + Assert.That(reader.Entries.Select(e => e.Name), Is.EqualTo(new[] { "content.bin" })); + } + + [Test, Category("Interop")] + public void PassphraseSignature_MatchesEcryptfsUtilsKnownAnswerVector() { + // ecryptfs-utils tests/userspace/verify-passphrase-sig.sh: + // pass="foo", default salt 0011223344556677 => 253ca7e88811d184. + using var image = new MemoryStream(CreateImage([1, 2, 3], password: "foo"), writable: false); + using var reader = new EcryptfsReader(image); + Assert.That(reader.PassphraseSignature, Is.EqualTo("253ca7e88811d184")); + } + + [Test, Category("Sad")] + public void Read_BrokenMarkerRelation_Throws() { + var bytes = CreateImage([1, 2, 3]); + bytes[12] ^= 0x80; + using var image = new MemoryStream(bytes, writable: false); + Assert.Throws(() => _ = new EcryptfsReader(image)); } [Test, Category("Sad")] - public void Read_BadMarker_Throws() { - var img = new byte[64]; - using var ms = new MemoryStream(img); - Assert.Throws(() => _ = new FileSystem.Ecryptfs.EcryptfsReader(ms)); + public void Read_TruncatedCiphertext_Throws() { + var bytes = CreateImage(new byte[4097]); + Array.Resize(ref bytes, bytes.Length - 1); + using var image = new MemoryStream(bytes, writable: false); + Assert.Throws(() => _ = new EcryptfsReader(image)); } } diff --git a/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs b/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs index 52dec651c..ba7336808 100644 --- a/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs +++ b/Compression.Tests/Ecryptfs/EcryptfsStubBehaviorTests.cs @@ -1,78 +1,176 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; +using System.Security.Cryptography; using Compression.Registry; using FileSystem.Ecryptfs; namespace Compression.Tests.Ecryptfs; -/// -/// Pins the stub-tier surface for . -/// eCryptfs per-file containers wrap AES-CBC ciphertext extents — without the -/// mount passphrase + EFEK packets the payload is opaque. These tests prevent -/// silent capability creep (CanCreate/CanModify) and stop the opaque-blob entry -/// shape from drifting. -/// [TestFixture] public class EcryptfsStubBehaviorTests { - private static byte[] BuildMagicOnly(int cipherLen = 256) { - // Reader places ciphertext at offset max(ExtentSize, 4096); we set extent=4096. - const int extent = 4096; - var image = new byte[extent + cipherLen]; - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(0, 4), 0x3C81B7F5u); - BinaryPrimitives.WriteUInt64BigEndian(image.AsSpan(4, 8), 8192ul); - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(12, 4), 0u); - BinaryPrimitives.WriteUInt32BigEndian(image.AsSpan(16, 4), (uint)extent); - for (var i = 0; i < cipherLen; i++) image[extent + i] = (byte)((i * 11) ^ 0xC3); - return image; + private static byte[] CreateImage(byte[] content, string password = "correct horse", string? method = null) { + var descriptor = new EcryptfsFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create( + image, + [ArchiveInputInfo.InMemory("payload.bin", content)], + new FormatCreateOptions { Password = password, EncryptionMethod = method }); + return image.ToArray(); } - [Test, Category("Stub")] - public void Stub_DescriptorHonestlyAdvertisesCapabilities_AndOpaqueEntries() { - var d = new EcryptfsFormatDescriptor(); - - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False, - "eCryptfs is stub-tier (encrypted ciphertext) — must not advertise CanCreate."); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False, - "eCryptfs is stub-tier (encrypted ciphertext) — must not advertise CanModify."); - - var image = BuildMagicOnly(cipherLen: 256); - using var ms = new MemoryStream(image, writable: false); - var entries = d.List(ms, null); - - var names = entries.Select(e => e.Name).ToList(); - Assert.That(names, Is.EquivalentTo(new[] { "FULL.ecryptfs", "metadata.ini", "ciphertext.bin" }), - "eCryptfs minimal-image surface must be exactly the documented opaque triple."); - - var outDir = Path.Combine(Path.GetTempPath(), "EcryptfsStub_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(outDir); - try { - using var ms2 = new MemoryStream(image, writable: false); - d.Extract(ms2, outDir, password: null, files: null); - - var fullPath = Path.Combine(outDir, "FULL.ecryptfs"); - Assert.That(File.Exists(fullPath), Is.True, "Extract must produce FULL.ecryptfs."); - Assert.That(File.ReadAllBytes(fullPath), Is.EqualTo(image), - "FULL.ecryptfs must round-trip the file bytes exactly."); - - var cipherPath = Path.Combine(outDir, "ciphertext.bin"); - Assert.That(File.Exists(cipherPath), Is.True, "Extract must produce ciphertext.bin."); - var expectedCipher = image.AsSpan(4096).ToArray(); - Assert.That(File.ReadAllBytes(cipherPath), Is.EqualTo(expectedCipher), - "ciphertext.bin must round-trip the opaque AES-CBC payload exactly."); - } finally { - Directory.Delete(outDir, recursive: true); + [TestCase("aes128")] + [TestCase("aes192")] + [TestCase("aes256")] + [Category("Interop")] + public void CreateThenExtract_RoundTripsAesPassphraseProfiles(string method) { + var content = Enumerable.Range(0, 9001).Select(i => (byte)(i * 73 + 11)).ToArray(); + var descriptor = new EcryptfsFormatDescriptor(); + using var image = new MemoryStream(CreateImage(content, method: method), writable: false); + + var entries = descriptor.List(image, null); + Assert.That(entries, Has.Count.EqualTo(1)); + Assert.That(entries[0].Name, Is.EqualTo("content.bin")); + Assert.That(entries[0].OriginalSize, Is.EqualTo(content.Length)); + Assert.That(entries[0].IsEncrypted, Is.True); + + image.Position = 0; + using var reader = new EcryptfsReader(image); + Assert.That(reader.ExtractContent("correct horse"), Is.EqualTo(content)); + } + + [Test, Category("RoundTrip")] + public void CredentialAwareAdd_ReplacesPayload_AndPreservesCipherProfile() { + const string password = "correct horse"; + var replacement = Enumerable.Range(0, 12001).Select(i => (byte)(i * 41 + 3)).ToArray(); + var descriptor = new EcryptfsFormatDescriptor(); + var modifier = (IArchiveModifiable)descriptor; + // Expandable: MemoryStream(byte[]) is fixed-capacity, and the replacement payload here is + // larger than the original, so the Add has to grow the image. + using var image = new MemoryStream(); + var seed = CreateImage([1, 2, 3, 4, 5], password, "aes256"); + image.Write(seed, 0, seed.Length); + + image.Position = 0; + string? signatureBefore; + using (var before = new EcryptfsReader(image)) { + signatureBefore = before.PassphraseSignature; + Assert.That(before.CipherDescription, Is.EqualTo("AES-256-CBC")); } + + image.Position = 0; + modifier.Add( + image, + [ArchiveInputInfo.InMemory("content.bin", replacement)], + new ArchiveMutationOptions { Password = password }); + + image.Position = 0; + using var reader = new EcryptfsReader(image); + Assert.That(reader.CipherDescription, Is.EqualTo("AES-256-CBC")); + Assert.That(reader.PassphraseSignature, Is.EqualTo(signatureBefore)); + Assert.That(reader.ExtractContent(password), Is.EqualTo(replacement)); } - [Test, Category("Stub")] - public void Stub_DoesNotAdvertiseWriteCapability() { - var d = new EcryptfsFormatDescriptor(); - var description = d.Description.ToLowerInvariant(); - Assert.That( - description.Contains("stub") || description.Contains("opaque") - || description.Contains("skeleton") || description.Contains("detection"), - Is.True, - $"eCryptfs Description must honestly flag its stub/detection-only/opaque status. Got: '{d.Description}'."); + [Test, Category("Sad")] + public void CredentialAwareAdd_WrongPassphrase_LeavesOriginalByteIdentical() { + var descriptor = new EcryptfsFormatDescriptor(); + var modifier = (IArchiveModifiable)descriptor; + using var image = new MemoryStream(CreateImage(Enumerable.Range(0, 5000).Select(i => (byte)i).ToArray())); + var before = image.ToArray(); + + image.Position = 0; + Assert.Throws(() => modifier.Add( + image, + [ArchiveInputInfo.InMemory("content.bin", [9, 8, 7, 6])], + new ArchiveMutationOptions { Password = "wrong horse" })); + + Assert.That(image.ToArray(), Is.EqualTo(before)); + } + + [Test, Category("Sad")] + public void CredentialFreeAdd_IsRejectedInsteadOfGuessingPassword() { + var modifier = (IArchiveModifiable)new EcryptfsFormatDescriptor(); + using var image = new MemoryStream(CreateImage([1, 2, 3])); + Assert.Throws(() => modifier.Add( + image, + [ArchiveInputInfo.InMemory("content.bin", [4, 5, 6])])); + } + + [Test, Category("HappyPath")] + public void Remove_ContentBin_LeavesValidEmptyLowerFile() { + var descriptor = new EcryptfsFormatDescriptor(); + var modifier = (IArchiveModifiable)descriptor; + using var image = new MemoryStream(CreateImage(Enumerable.Repeat((byte)0x6D, 5000).ToArray())); + + modifier.Remove(image, ["content.bin"]); + + Assert.That(image.Length, Is.EqualTo(8192)); + image.Position = 0; + Assert.That(descriptor.List(image, null), Is.Empty); + using var reader = new EcryptfsReader(image); + reader.ValidatePassword("correct horse"); + } + + [Test, Category("Sad")] + public void Extract_WrongPassphrase_IsRejectedBeforePayloadDecryption() { + using var image = new MemoryStream(CreateImage([1, 2, 3, 4]), writable: false); + using var reader = new EcryptfsReader(image); + Assert.Throws(() => reader.ExtractContent("wrong horse")); + } + + [Test, Category("HappyPath")] + public void WipeAndShrink_RemoveOnlyProvenDeadBytes() { + var content = Enumerable.Range(0, 5000).Select(i => (byte)i).ToArray(); + var descriptor = new EcryptfsFormatDescriptor(); + var original = CreateImage(content); + using var image = new MemoryStream(); + image.Write(original); + image.Write(Enumerable.Repeat((byte)0xA5, 257).ToArray()); + image.Position = 0; + + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(image); + Assert.That(wiped, Is.GreaterThanOrEqualTo(257)); + Assert.That(image.ToArray().AsSpan(original.Length, 257).ToArray(), Is.All.Zero); + + image.Position = 0; + using (var reader = new EcryptfsReader(image)) + Assert.That(reader.ExtractContent("correct horse"), Is.EqualTo(content)); + + image.Position = 0; + using var shrunk = new MemoryStream(); + ((IArchiveShrinkable)descriptor).Shrink(image, shrunk); + Assert.That(shrunk.Length, Is.EqualTo(original.Length)); + using var shrunkReader = new EcryptfsReader(shrunk); + Assert.That(shrunkReader.ExtractContent("correct horse"), Is.EqualTo(content)); + } + + [Test, Category("HappyPath")] + public void Purge_LeavesValidListableEmptyLowerFile() { + var descriptor = new EcryptfsFormatDescriptor(); + using var image = new MemoryStream(CreateImage(Enumerable.Repeat((byte)0x5A, 5000).ToArray())); + + ((IArchivePurgeable)descriptor).Purge(image); + + Assert.That(image.Length, Is.EqualTo(8192)); + image.Position = 0; + Assert.That(descriptor.List(image, null), Is.Empty); + image.Position = 0; + using var reader = new EcryptfsReader(image); + Assert.That(reader.DecryptedSize, Is.Zero); + Assert.That(reader.CanonicalLength, Is.EqualTo(8192)); + } + + [Test, Category("Sad")] + public void Create_RequiresPassphrase_AndSingleRegularInput() { + var descriptor = (IArchiveCreatable)new EcryptfsFormatDescriptor(); + using var output = new MemoryStream(); + Assert.Throws(() => descriptor.Create( + output, + [ArchiveInputInfo.InMemory("a", [1])], + new FormatCreateOptions())); + + Assert.Throws(() => descriptor.Create( + output, + [ArchiveInputInfo.InMemory("a", [1]), ArchiveInputInfo.InMemory("b", [2])], + new FormatCreateOptions { Password = "pw" })); } } diff --git a/Compression.Tests/Ext/ExtExtendedAttributesTests.cs b/Compression.Tests/Ext/ExtExtendedAttributesTests.cs new file mode 100644 index 000000000..e106935ea --- /dev/null +++ b/Compression.Tests/Ext/ExtExtendedAttributesTests.cs @@ -0,0 +1,70 @@ +using FileSystem.Ext; + +namespace Compression.Tests.Ext; + +[TestFixture] +public class ExtExtendedAttributesTests { + + private static MemoryStream BuildImage() { + var writer = new ExtWriter(); + writer.AddFile("brick/data.txt", "payload"u8.ToArray()); + var bytes = writer.Build( + blockSize: 1024, + totalBlocks: 4096, + version: ExtWriter.ExtVersion.Ext4, + journal: false, + volumeLabel: "xattr-test", + inodeSize: 256); + return new MemoryStream(bytes, writable: true); + } + + [Test, Category("RoundTrip")] + public void TrustedGfid_CreateUpdateRemove_RoundTrips() { + using var image = BuildImage(); + byte[] first = [0x10, 0x32, 0x54, 0x76, 0x98, 0xBA, 0xDC, 0xFE, 1, 2, 3, 4, 5, 6, 7, 8]; + byte[] second = [8, 7, 6, 5, 4, 3, 2, 1, 0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10]; + + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", first); + Assert.That(ExtExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(first)); + + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", second); + Assert.That(ExtExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(second)); + + Assert.That(ExtExtendedAttributes.Remove(image, "brick/data.txt", "trusted.gfid"), Is.True); + Assert.That(ExtExtendedAttributes.Read(image, "brick/data.txt").ContainsKey("trusted.gfid"), Is.False); + Assert.That(ExtExtendedAttributes.Remove(image, "brick/data.txt", "trusted.gfid"), Is.False); + } + + [Test, Category("RoundTrip")] + public void MultipleTrustedAttributes_RoundTripWithoutDisturbingPayload() { + using var image = BuildImage(); + var gfid = Enumerable.Range(0, 16).Select(i => (byte)i).ToArray(); + var dht = Enumerable.Range(16, 16).Select(i => (byte)i).ToArray(); + + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", gfid); + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.glusterfs.dht", dht); + + var attributes = ExtExtendedAttributes.Read(image, "brick/data.txt"); + Assert.Multiple(() => { + Assert.That(attributes["trusted.gfid"], Is.EqualTo(gfid)); + Assert.That(attributes["trusted.glusterfs.dht"], Is.EqualTo(dht)); + }); + + image.Position = 0; + using var reader = new ExtReader(image, leaveOpen: true); + var entry = reader.Entries.Single(entry => entry.Name == "brick/data.txt"); + Assert.That(reader.Extract(entry), Is.EqualTo("payload"u8.ToArray())); + } + + [Test, Category("ErrorHandling")] + public void OversizedInlineSet_FailsWithoutChangingExistingAttribute() { + using var image = BuildImage(); + var gfid = Enumerable.Range(0, 16).Select(i => (byte)i).ToArray(); + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", gfid); + + Assert.Throws(() => + ExtExtendedAttributes.Set(image, "brick/data.txt", "trusted.too-large", new byte[128])); + + Assert.That(ExtExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(gfid)); + } +} diff --git a/Compression.Tests/FirmwareHex/FirmwareContainerWriteTests.cs b/Compression.Tests/FirmwareHex/FirmwareContainerWriteTests.cs index e0520427a..041f9c319 100644 --- a/Compression.Tests/FirmwareHex/FirmwareContainerWriteTests.cs +++ b/Compression.Tests/FirmwareHex/FirmwareContainerWriteTests.cs @@ -10,11 +10,10 @@ namespace Compression.Tests.FirmwareHex; /// round trip. /// /// -/// Intel HEX, TI-TXT and the legacy uImage are all fully specified and all three -/// were read-only, which made three tracked gaps in the support matrix that were -/// no harder to close than transcribing the specification. Each is a single -/// payload under a header or a base address, so they are WORM rather than R/W — -/// there is no second file to add to one. +/// Intel HEX, TI-TXT and the legacy uImage are all fully specified and originally +/// had read-only descriptors. Their creators close the write side by transcribing +/// those specifications. Intel HEX additionally supports rebuild-based editing; +/// TI-TXT and uImage remain WORM because they expose only a single payload view. /// [TestFixture] [Category("HappyPath"), Category("RoundTrip")] diff --git a/Compression.Tests/FirmwareHex/IntelHexRwTests.cs b/Compression.Tests/FirmwareHex/IntelHexRwTests.cs new file mode 100644 index 000000000..251df01d1 --- /dev/null +++ b/Compression.Tests/FirmwareHex/IntelHexRwTests.cs @@ -0,0 +1,141 @@ +#pragma warning disable CS1591 +using System.Text; +using Compression.Registry; +using FileFormat.FirmwareHex; + +namespace Compression.Tests.FirmwareHex; + +[TestFixture] +public sealed class IntelHexRwTests { + + private const string SparseHex = + ":0410000001020304E2\n" + + ":03200000AABBCCAC\n" + + ":0400000312345678E5\n" + + ":00000001FF\n"; + + [Test, Category("HappyPath")] + public void Reader_DecodesType03AsCsShift4PlusIp_AndKeepsRegisters() { + var image = IntelHexReader.Read(":0400000312345678E5\n:00000001FF\n"); + + Assert.Multiple(() => { + Assert.That(image.StartAddress, Is.EqualTo(0x000179B8u)); + Assert.That(image.StartSegmentAddress, Is.Not.Null); + Assert.That(image.StartSegmentAddress?.CodeSegment, Is.EqualTo((ushort)0x1234)); + Assert.That(image.StartSegmentAddress?.InstructionPointer, Is.EqualTo((ushort)0x5678)); + }); + } + + [TestCase(":0100000412E9\n:00000001FF\n", TestName = "Reader_RejectsWrongExtendedLinearPayloadLength")] + [TestCase(":020001040001F8\n:00000001FF\n", TestName = "Reader_RejectsNonZeroExtendedLinearAddressField")] + [TestCase(":00000101FE\n", TestName = "Reader_RejectsNonZeroEofAddressField")] + [TestCase(":00000001FF\n:00000001FF\n", TestName = "Reader_RejectsRecordAfterEof")] + [Category("EdgeCase")] + public void Reader_RejectsMalformedControlRecords(string text) + => Assert.That(() => IntelHexReader.Read(text), Throws.InstanceOf()); + + [Test, Category("RoundTrip")] + public void ExtractCreateRoundTrip_PreservesSparseRunsAndSegmentedStartRecord() { + var descriptor = new IntelHexFormatDescriptor(); + var extracted = Extract(descriptor, Encoding.ASCII.GetBytes(SparseHex)); + + using var rebuilt = new MemoryStream(); + descriptor.Create(rebuilt, + extracted.Select(pair => ArchiveInputInfo.InMemory(pair.Key, pair.Value)).ToArray(), + new FormatCreateOptions()); + + var text = Encoding.ASCII.GetString(rebuilt.ToArray()); + var image = IntelHexReader.Read(text); + Assert.Multiple(() => { + Assert.That(image.Segments, Has.Count.EqualTo(2), "the address hole was flattened into programmed 0xFF bytes"); + Assert.That(image.Segments[0].Address, Is.EqualTo(0x00001000u)); + Assert.That(image.Segments[0].Data, Is.EqualTo(new byte[] { 1, 2, 3, 4 }).AsCollection); + Assert.That(image.Segments[1].Address, Is.EqualTo(0x00002000u)); + Assert.That(image.Segments[1].Data, Is.EqualTo(new byte[] { 0xAA, 0xBB, 0xCC }).AsCollection); + Assert.That(image.StartSegmentAddress?.CodeSegment, Is.EqualTo((ushort)0x1234)); + Assert.That(image.StartSegmentAddress?.InstructionPointer, Is.EqualTo((ushort)0x5678)); + Assert.That(text, Does.Contain(":0400000312345678E5"), "type-03 CS:IP was rewritten as a type-05 linear start"); + }); + } + + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Descriptor_AdvertisesRw_AndCanReplaceItsFirmwareView() { + var descriptor = new IntelHexFormatDescriptor(); + Assert.Multiple(() => { + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + }); + + var original = Create(descriptor, [ArchiveInputInfo.InMemory("firmware.bin", new byte[] { 1, 2, 3, 4 })]); + using var archive = Expandable(original); + ((IArchiveModifiable)descriptor).Add(archive, + [ArchiveInputInfo.InMemory("firmware.bin", new byte[] { 9, 8, 7, 6 })]); + + var extracted = Extract(descriptor, archive.ToArray()); + Assert.That(extracted["firmware.bin"], Is.EqualTo(new byte[] { 9, 8, 7, 6 }).AsCollection); + } + + [Test, Category("RoundTrip")] + public void Defragment_RebuildsCanonicallyWithoutProgrammingSparseHole() { + var descriptor = new IntelHexFormatDescriptor(); + using var archive = Expandable(Encoding.ASCII.GetBytes(SparseHex)); + + ((IArchiveDefragmentable)descriptor).Defragment(archive); + + archive.Position = 0; + using var reader = new StreamReader(archive, Encoding.ASCII, detectEncodingFromByteOrderMarks: true, leaveOpen: true); + var image = IntelHexReader.Read(reader.ReadToEnd()); + Assert.Multiple(() => { + Assert.That(image.Segments, Has.Count.EqualTo(2)); + Assert.That(image.Segments[0].Address, Is.EqualTo(0x00001000u)); + Assert.That(image.Segments[1].Address, Is.EqualTo(0x00002000u)); + Assert.That(image.TotalDataBytes, Is.EqualTo(7)); + Assert.That(image.StartSegmentAddress, Is.Not.Null); + }); + } + + [Test, Category("HappyPath")] + public void Purge_LeavesCanonicalEmptyIntelHex() { + var descriptor = new IntelHexFormatDescriptor(); + using var archive = Expandable(Encoding.ASCII.GetBytes(SparseHex)); + + ((IArchivePurgeable)descriptor).Purge(archive); + + var text = Encoding.ASCII.GetString(archive.ToArray()); + var image = IntelHexReader.Read(text); + Assert.Multiple(() => { + Assert.That(text.Trim(), Is.EqualTo(":00000001FF")); + Assert.That(image.Segments, Is.Empty); + Assert.That(image.StartAddress, Is.Null); + Assert.That(image.StartSegmentAddress, Is.Null); + }); + } + + private static byte[] Create(IntelHexFormatDescriptor descriptor, IReadOnlyList inputs) { + using var output = new MemoryStream(); + descriptor.Create(output, inputs, new FormatCreateOptions()); + return output.ToArray(); + } + + private static MemoryStream Expandable(byte[] data) { + var result = new MemoryStream(); + result.Write(data); + result.Position = 0; + return result; + } + + private static Dictionary Extract(IntelHexFormatDescriptor descriptor, byte[] image) { + var directory = Path.Combine(Path.GetTempPath(), "cwb_ihex_" + Guid.NewGuid().ToString("N")[..10]); + Directory.CreateDirectory(directory); + try { + using var input = new MemoryStream(image, writable: false); + descriptor.Extract(input, directory, null, null); + return Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories) + .ToDictionary(path => Path.GetFileName(path)!, File.ReadAllBytes, StringComparer.OrdinalIgnoreCase); + } finally { + try { Directory.Delete(directory, recursive: true); } catch { /* best effort */ } + } + } +} diff --git a/Compression.Tests/FirmwareHex/TiTxtMaintenanceTests.cs b/Compression.Tests/FirmwareHex/TiTxtMaintenanceTests.cs new file mode 100644 index 000000000..50aa144d2 --- /dev/null +++ b/Compression.Tests/FirmwareHex/TiTxtMaintenanceTests.cs @@ -0,0 +1,129 @@ +using System.Text; +using Compression.Registry; +using FileFormat.FirmwareHex; + +namespace Compression.Tests.FirmwareHex; + +[TestFixture] +public sealed class TiTxtMaintenanceTests { + + [Test] + public void Parser_RejectsOverlappingSections() { + const string text = "@1000\n01 02 03 04\n@1002\nAA BB\nq\n"; + Assert.That(() => TiTxtReader.Read(text), Throws.InstanceOf()); + } + + [Test] + public void Parser_RejectsTrailingContentAfterTerminator() { + const string text = "@1000\n01 02\nq\n@2000\n03 04\n"; + Assert.That(() => TiTxtReader.Read(text), Throws.InstanceOf()); + } + + [Test] + public void Defragment_CanonicalisesAdjacentSectionsWithoutFillingHoles() { + const string text = "@1010\n05 06\n@1004\n03 04\n@1000\n01 02 07 08\nq\n"; + using var image = StreamOf(text); + var descriptor = new TiTxtFormatDescriptor(); + + descriptor.Defragment(image); + + var rewritten = Encoding.ASCII.GetString(image.ToArray()); + var parsed = TiTxtReader.Read(rewritten); + Assert.Multiple(() => { + Assert.That(parsed.Segments, Has.Count.EqualTo(2)); + Assert.That(parsed.Segments[0].Address, Is.EqualTo(0x1000u)); + Assert.That(parsed.Segments[0].Data, Is.EqualTo(new byte[] { 1, 2, 7, 8, 3, 4 }).AsCollection); + Assert.That(parsed.Segments[1].Address, Is.EqualTo(0x1010u)); + Assert.That(parsed.Segments[1].Data, Is.EqualTo(new byte[] { 5, 6 }).AsCollection); + Assert.That(rewritten, Does.Not.Contain("@1004")); + Assert.That(rewritten, Does.Contain("@1010")); + }); + } + + [Test] + public void Add_ReplacesPayloadAtExistingBaseAddress() { + using var image = StreamOf("@C000\n01 02 03\nq\n"); + var descriptor = new TiTxtFormatDescriptor(); + + descriptor.Add(image, [ArchiveInputInfo.InMemory("replacement.bin", new byte[] { 0xDE, 0xAD, 0xBE, 0xEF })]); + + var parsed = Parse(image); + Assert.Multiple(() => { + Assert.That(parsed.Segments, Has.Count.EqualTo(1)); + Assert.That(parsed.Segments[0].Address, Is.EqualTo(0xC000u)); + Assert.That(parsed.Segments[0].Data, Is.EqualTo(new byte[] { 0xDE, 0xAD, 0xBE, 0xEF }).AsCollection); + }); + } + + [Test] + public void Add_MetadataOnlyShiftPreservesSparseGaps() { + using var image = StreamOf("@1000\n01 02\n@1010\n03 04\nq\n"); + var descriptor = new TiTxtFormatDescriptor(); + var metadata = Encoding.UTF8.GetBytes("[firmware_hex]\nbase_address = 0x00002000\n"); + + descriptor.Add(image, [ArchiveInputInfo.InMemory(FirmwareHexWriter.MetadataName, metadata)]); + + var parsed = Parse(image); + Assert.Multiple(() => { + Assert.That(parsed.Segments, Has.Count.EqualTo(2)); + Assert.That(parsed.Segments[0].Address, Is.EqualTo(0x2000u)); + Assert.That(parsed.Segments[1].Address, Is.EqualTo(0x2010u)); + Assert.That(parsed.GapCount, Is.EqualTo(1)); + Assert.That(parsed.TotalDataBytes, Is.EqualTo(4)); + }); + } + + [Test] + public void Remove_FirmwareLeavesValidEmptyImage() { + using var image = StreamOf("@1000\n01 02\nq\n"); + var descriptor = new TiTxtFormatDescriptor(); + + descriptor.Remove(image, [FirmwareHexWriter.PayloadName]); + + Assert.Multiple(() => { + Assert.That(Encoding.ASCII.GetString(image.ToArray()), Is.EqualTo("q\n")); + Assert.That(Parse(image).Segments, Is.Empty); + }); + } + + [Test] + public void Purge_RemovesLivePayloadAndLeavesValidEmptyImage() { + using var image = StreamOf("@1000\n01 02\n@2000\n03 04\nq\n"); + var descriptor = new TiTxtFormatDescriptor(); + + descriptor.Purge(image); + + Assert.Multiple(() => { + Assert.That(Encoding.ASCII.GetString(image.ToArray()), Is.EqualTo("q\n")); + Assert.That(Parse(image).Segments, Is.Empty); + }); + } + + [Test] + public void Create_RejectsMultiplePayloads() { + var descriptor = new TiTxtFormatDescriptor(); + using var output = new MemoryStream(); + var inputs = new[] { + ArchiveInputInfo.InMemory("a.bin", new byte[] { 1 }), + ArchiveInputInfo.InMemory("b.bin", new byte[] { 2 }), + }; + + Assert.That(() => descriptor.Create(output, inputs, new FormatCreateOptions()), + Throws.InstanceOf()); + } + + // Built empty and written into: MemoryStream(byte[]) is fixed-capacity, and the maintenance + // verbs under test grow the image. + private static MemoryStream StreamOf(string text) { + var result = new MemoryStream(); + var bytes = Encoding.ASCII.GetBytes(text); + result.Write(bytes, 0, bytes.Length); + result.Position = 0; + return result; + } + + private static FirmwareImage Parse(MemoryStream stream) { + stream.Position = 0; + return TiTxtReader.Read(Encoding.ASCII.GetString(stream.ToArray())); + } +} diff --git a/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs b/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs index 02a16ffbc..98cfce84b 100644 --- a/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs +++ b/Compression.Tests/GlusterFs/GlusterFsDetectionTests.cs @@ -1,84 +1,199 @@ using System.Text; using Compression.Registry; +using FileSystem.Ext; using FileSystem.GlusterFs; +using FileSystem.Xfs; namespace Compression.Tests.GlusterFs; [TestFixture] public class GlusterFsDetectionTests { - private static byte[] BuildMinimal(int payloadLen = 128) { - var image = new byte[8 + payloadLen]; - image[0] = 0xCA; image[1] = 0xFE; image[2] = 0x5B; image[3] = 0xAB; - for (var i = 0; i < payloadLen; i++) image[8 + i] = (byte)(i & 0xFF); - return image; + private static readonly byte[] Gfid = [ + 0x10, 0x32, 0x54, 0x76, 0x98, 0xBA, 0xDC, 0xFE, + 0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, + ]; + + private static byte[] BuildXfsBrick() { + var writer = new XfsWriter(); + writer.AddFile("host/brick/data/hello.txt", "hello from xfs brick"u8.ToArray()); + writer.AddFile("host/brick/.glusterfs/ab/internal-gfid", "backend index"u8.ToArray()); + writer.AddFile("outside.txt", "not part of the brick"u8.ToArray()); + using var stream = new MemoryStream(); + writer.WriteTo(stream); + XfsExtendedAttributes.Set(stream, "host/brick/data/hello.txt", "trusted.gfid", Gfid); + return stream.ToArray(); + } + + private static byte[] BuildExtBrick() { + var writer = new ExtWriter(); + writer.AddFile("srv/brick/data/hello.txt", "hello from ext brick"u8.ToArray()); + writer.AddFile("srv/brick/.glusterfs/cd/internal-gfid", "backend index"u8.ToArray()); + writer.AddFile("outside.txt", "not part of the brick"u8.ToArray()); + var bytes = writer.Build( + blockSize: 1024, + totalBlocks: 4096, + version: ExtWriter.ExtVersion.Ext4, + journal: false, + volumeLabel: "gluster", + inodeSize: 256); + using var stream = new MemoryStream(bytes, writable: true); + ExtExtendedAttributes.Set(stream, "srv/brick/data/hello.txt", "trusted.gfid", Gfid); + return stream.ToArray(); } [Test, Category("HappyPath")] - public void Detector_IdentifiesByMagic() { - var d = new GlusterFsFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("GlusterFs")); - Assert.That(d.Extensions, Does.Contain(".gluster")); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(1)); - Assert.That(d.MagicSignatures[0].Bytes, Is.EqualTo(new byte[] { 0xCA, 0xFE, 0x5B, 0xAB })); - Assert.That(d, Is.Not.InstanceOf()); + public void Descriptor_UsesExtensionOnlyDetectionAndAdvertisesReadOnlyBrickView() { + var descriptor = new GlusterFsFormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor.Id, Is.EqualTo("GlusterFs")); + Assert.That(descriptor.Extensions, Does.Contain(".gluster")); + Assert.That(descriptor.MagicSignatures, Is.Empty, + "GlusterFS has no standalone block-format magic and must not collide with XFS/ext detection."); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanList), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanExtract), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.SupportsDirectories), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + Assert.That(descriptor, Is.InstanceOf()); + }); } [Test, Category("HappyPath")] - public void List_ReturnsTwoEntries() { - var d = new GlusterFsFormatDescriptor(); - using var ms = new MemoryStream(BuildMinimal(payloadLen: 256)); - var entries = d.List(ms, password: null); - var names = entries.Select(e => e.Name).ToList(); - Assert.That(names, Is.EquivalentTo(new[] { "metadata.ini", "gluster-brick.bin" })); + public void XfsBackingStore_InfersBrickRootAndHidesNonBrickContentAndGlusterIndex() { + var descriptor = new GlusterFsFormatDescriptor(); + using var stream = new MemoryStream(BuildXfsBrick()); + + var names = descriptor.List(stream, password: null).Select(entry => entry.Name).ToArray(); + + Assert.Multiple(() => { + Assert.That(names, Does.Contain("metadata.ini")); + Assert.That(names, Does.Contain("brick/data/hello.txt")); + Assert.That(names.Any(name => name.Contains(".glusterfs", StringComparison.Ordinal)), Is.False); + Assert.That(names, Does.Not.Contain("brick/outside.txt")); + }); } - [Test, Category("Stub")] - public void Description_FlagsStage0Permanent() { - var d = new GlusterFsFormatDescriptor(); - var lower = d.Description.ToLowerInvariant(); - Assert.That(lower, Does.Contain("detection-only")); - Assert.That(lower, Does.Contain("stage 0")); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + [Test, Category("HappyPath")] + public void ExtBackingStore_InfersBrickRootAndExtractsBrickFile() { + using var stream = new MemoryStream(BuildExtBrick()); + using var reader = new GlusterFsReader(stream); + + var entry = reader.Entries.Single(candidate => candidate.Name == "brick/data/hello.txt"); + + Assert.Multiple(() => { + Assert.That(reader.ValidHeader, Is.True); + Assert.That(reader.BackingFileSystem, Is.EqualTo("ext")); + Assert.That(reader.BrickRoot, Is.EqualTo("srv/brick")); + Assert.That(reader.HasGlusterIndex, Is.True); + Assert.That(Encoding.UTF8.GetString(reader.Extract(entry)), Is.EqualTo("hello from ext brick")); + Assert.That(reader.ReadExtendedAttributes(entry)["trusted.gfid"], Is.EqualTo(Gfid)); + Assert.That(reader.Entries.Any(candidate => candidate.Name.Contains(".glusterfs", StringComparison.Ordinal)), Is.False); + Assert.That(reader.Entries.Any(candidate => candidate.Name == "brick/outside.txt"), Is.False); + }); } - [Test, Category("Stub")] - public void Description_DocumentsHonestFallback() { - // The honest-fallback rule: a GlusterFS brick is a normal directory on a - // local POSIX FS and state lives in xattrs — no on-disk image, no R/O - // promotion possible from a single image stream. - var d = new GlusterFsFormatDescriptor(); - var lower = d.Description.ToLowerInvariant(); - Assert.That(lower, Does.Contain("brick")); - Assert.That(lower, Does.Contain("director")); // "directory" or "directories" - Assert.That(lower, Does.Contain("xattr")); - Assert.That(lower, Does.Contain("no on-disk image")); - Assert.That(lower, Does.Contain("probe")); // magic is workbench probe, not real + [Test, Category("HappyPath")] + public void XfsBackingStore_SurfacesNativeGlusterXattrs() { + using var stream = new MemoryStream(BuildXfsBrick()); + using var reader = new GlusterFsReader(stream); + var entry = reader.Entries.Single(candidate => candidate.Name == "brick/data/hello.txt"); + + Assert.That(reader.ReadExtendedAttributes(entry)["trusted.gfid"], Is.EqualTo(Gfid)); } - [Test, Category("Stub")] - public void Metadata_DeclaresStage0PermanentAndExplainsWhyROBlocked() { - // The synthetic metadata.ini must spell out (a) stage=0, (b) stage is - // permanent, (c) the concrete reason R/O promotion is blocked. This is the - // contract the parent-task pipeline checks against when triaging FSes. - var d = new GlusterFsFormatDescriptor(); - using var ms = new MemoryStream(BuildMinimal(payloadLen: 64)); - var workDir = Path.Combine(Path.GetTempPath(), $"GlusterFsMeta_{Guid.NewGuid():N}"); + [Test, Category("HappyPath")] + public void XfsBackingStore_ExtractsThroughDescriptor() { + var descriptor = new GlusterFsFormatDescriptor(); + using var stream = new MemoryStream(BuildXfsBrick()); + var workDir = Path.Combine(Path.GetTempPath(), $"GlusterFsBrick_{Guid.NewGuid():N}"); Directory.CreateDirectory(workDir); try { - d.Extract(ms, workDir, password: null, files: ["metadata.ini"]); - var path = Path.Combine(workDir, "metadata.ini"); - Assert.That(File.Exists(path), Is.True); - var text = File.ReadAllText(path, Encoding.UTF8); - Assert.That(text, Does.Contain("stage=0")); - Assert.That(text, Does.Contain("stage_permanent=true")); - Assert.That(text, Does.Contain("ro_blocked_reason=")); - Assert.That(text, Does.Contain("xattr")); - Assert.That(text, Does.Contain("no on-disk image format")); - Assert.That(text, Does.Contain("magic_kind=workbench-internal probe")); + descriptor.Extract(stream, workDir, password: null, files: ["brick/data/hello.txt"]); + var path = Path.Combine(workDir, "brick", "data", "hello.txt"); + Assert.That(File.ReadAllText(path, Encoding.UTF8), Is.EqualTo("hello from xfs brick")); } finally { try { Directory.Delete(workDir, recursive: true); } catch { /* best effort */ } } } + + [Test, Category("RoundTrip")] + public void ExtBackingStore_ShrinkToFit_PreservesPayloadAndGfid() { + var original = BuildExtBrick(); + var descriptor = new GlusterFsFormatDescriptor(); + using var input = new MemoryStream(original, writable: false); + using var output = new MemoryStream(); + + descriptor.Shrink(input, output); + + Assert.That(output.Length, Is.LessThan(original.LongLength)); + output.Position = 0; + using var reader = new GlusterFsReader(output); + var entry = reader.Entries.Single(candidate => candidate.Name == "brick/data/hello.txt"); + Assert.Multiple(() => { + Assert.That(Encoding.UTF8.GetString(reader.Extract(entry)), Is.EqualTo("hello from ext brick")); + Assert.That(reader.ReadExtendedAttributes(entry)["trusted.gfid"], Is.EqualTo(Gfid)); + }); + } + + [Test, Category("RoundTrip")] + public void XfsBackingStore_ShrinkCopiesThroughWithoutRebuild() { + var original = BuildXfsBrick(); + var descriptor = new GlusterFsFormatDescriptor(); + using var input = new MemoryStream(original, writable: false); + using var output = new MemoryStream(); + + descriptor.Shrink(input, output); + + Assert.That(output.ToArray(), Is.EqualTo(original)); + output.Position = 0; + using var reader = new GlusterFsReader(output); + var entry = reader.Entries.Single(candidate => candidate.Name == "brick/data/hello.txt"); + Assert.That(reader.ReadExtendedAttributes(entry)["trusted.gfid"], Is.EqualTo(Gfid)); + } + + [Test, Category("Regression")] + public void WorkbenchProbeMagic_IsNoLongerAcceptedAsGlusterFs() { + byte[] oldSyntheticProbe = [0xCA, 0xFE, 0x5B, 0xAB, 0, 0, 0, 0]; + using var stream = new MemoryStream(oldSyntheticProbe); + + var error = Assert.Throws(() => new GlusterFsReader(stream)); + + Assert.That(error!.Message, Does.Contain("backing-store image")); + } + + [Test, Category("Regression")] + public void Descriptor_DoesNotAdvertiseUnsafeMutations() { + var descriptor = new GlusterFsFormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + }); + } + + [Test, Category("HappyPath")] + public void Metadata_DescribesXattrProgressAndClusterOperationBoundary() { + using var stream = new MemoryStream(BuildXfsBrick()); + using var reader = new GlusterFsReader(stream); + var metadata = reader.Entries.Single(entry => entry.Name == "metadata.ini"); + var text = Encoding.UTF8.GetString(reader.Extract(metadata)); + + Assert.Multiple(() => { + Assert.That(text, Does.Contain("parse_status=single-brick-read-only")); + Assert.That(text, Does.Contain("backing_fs=xfs")); + Assert.That(text, Does.Contain("brick_root=host/brick")); + Assert.That(text, Does.Contain("gluster_index_detected=true")); + Assert.That(text, Does.Contain("xattrs=readable through native backing-filesystem accessors")); + Assert.That(text, Does.Contain("xattr_mutation=native inline/short-form subset only")); + Assert.That(text, Does.Contain("cluster_namespace_reconstruction=false")); + Assert.That(text, Does.Contain("cluster_operations=rebalance,fix-layout,remove-brick are outside the single-image abstraction")); + Assert.That(text, Does.Contain("external/leaf/btree xattr mutation is not complete yet")); + }); + } } diff --git a/Compression.Tests/Ipsw/IpswMaintenanceTests.cs b/Compression.Tests/Ipsw/IpswMaintenanceTests.cs new file mode 100644 index 000000000..b4c72815e --- /dev/null +++ b/Compression.Tests/Ipsw/IpswMaintenanceTests.cs @@ -0,0 +1,180 @@ +using System.IO.Compression; +using Compression.Registry; +using FileFormat.Ipsw; + +namespace Compression.Tests.Ipsw; + +[TestFixture] +public class IpswMaintenanceTests { + + [Test, Category("HappyPath")] + public void Descriptor_AdvertisesFeasibleMaintenanceContracts() { + var descriptor = new IpswFormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + }); + } + + [Test, Category("RoundTrip")] + public void Defragment_RemovesDeadZipSpaceAndPreservesLiveMembers() { + var descriptor = new IpswFormatDescriptor(); + using var image = BuildIpsw( + ("BuildManifest.plist", ""u8.ToArray()), + ("Firmware/dead.bin", Enumerable.Repeat((byte)0xA5, 64 * 1024).ToArray()), + ("Firmware/dfu/iBSS.test.RELEASE.im4p", [0xDE, 0xAD, 0xBE, 0xEF])); + + ((IArchiveModifiable)descriptor).Remove(image, ["Firmware/dead.bin"]); + var beforeLength = image.Length; + + ((IArchiveDefragmentable)descriptor).Defragment(image); + + Assert.That(image.Length, Is.LessThan(beforeLength)); + Assert.That(ReadEntry(image, "Firmware/dfu/iBSS.test.RELEASE.im4p"), Is.EqualTo(new byte[] { 0xDE, 0xAD, 0xBE, 0xEF })); + Assert.That(RawNames(image), Does.Not.Contain("Firmware/dead.bin")); + Assert.That(RawNames(image), Does.Contain("Firmware/dfu/iBSS.test.RELEASE.im4p")); + } + + [Test, Category("RoundTrip")] + public void Shrink_TightPacksDeadZipSpaceAndPreservesLiveMembers() { + var descriptor = new IpswFormatDescriptor(); + using var image = BuildIpsw( + ("BuildManifest.plist", ""u8.ToArray()), + ("Firmware/dead.bin", Enumerable.Repeat((byte)0x5A, 64 * 1024).ToArray()), + ("Restore.plist", "restore"u8.ToArray())); + ((IArchiveModifiable)descriptor).Remove(image, ["Firmware/dead.bin"]); + + using var output = new MemoryStream(); + ((IArchiveShrinkable)descriptor).Shrink(image, output); + + Assert.That(output.Length, Is.LessThan(image.Length)); + Assert.That(ReadEntry(output, "Restore.plist"), Is.EqualTo("restore"u8.ToArray())); + Assert.That(RawNames(output), Is.EquivalentTo(new[] { "BuildManifest.plist", "Restore.plist" })); + } + + [Test, Category("RoundTrip")] + public void Purge_LeavesAValidEmptyIpswContainer() { + var descriptor = new IpswFormatDescriptor(); + using var image = BuildIpsw( + ("BuildManifest.plist", ""u8.ToArray()), + ("Firmware/dfu/iBSS.test.RELEASE.im4p", [1, 2, 3, 4])); + + ((IArchivePurgeable)descriptor).Purge(image); + + Assert.That(RawNames(image), Is.Empty); + image.Position = 0; + var listed = descriptor.List(image, null).Select(entry => entry.Name).ToArray(); + Assert.That(listed, Is.EquivalentTo(new[] { "FULL.ipsw", "metadata.ini" })); + } + + [Test, Category("RoundTrip")] + public void Wipe_PreservesZipDataDescriptors() { + var original = BuildDataDescriptorIpsw(); + using var image = new MemoryStream(original.ToArray()); + var descriptor = new IpswFormatDescriptor(); + + var layout = ((IArchiveLayoutMap)descriptor).EnumerateLayout(image).ToArray(); + Assert.That(layout, Has.Some.Matches(block => + block.Kind == DefragBlockKind.MetadataReserved && + block.FileName?.StartsWith("Data descriptor:", StringComparison.Ordinal) == true)); + + image.Position = 0; + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(image); + + Assert.Multiple(() => { + Assert.That(wiped, Is.Zero); + Assert.That(image.ToArray(), Is.EqualTo(original)); + }); + } + + [Test, Category("RoundTrip")] + public void Create_FileBackedInputStreamsPayloadAndPreservesNestedPath() { + var temp = Path.GetTempFileName(); + try { + var payload = Enumerable.Range(0, 1024 * 1024).Select(i => (byte)(i * 31)).ToArray(); + File.WriteAllBytes(temp, payload); + var descriptor = new IpswFormatDescriptor(); + using var image = new MemoryStream(); + + ((IArchiveCreatable)descriptor).Create(image, [ + ArchiveInputInfo.FromFile(new FileInfo(temp), "Firmware/dfu/streamed.bin"), + ], new FormatCreateOptions()); + + Assert.That(ReadEntry(image, "Firmware/dfu/streamed.bin"), Is.EqualTo(payload)); + image.Position = 0; + Assert.That(descriptor.List(image, null).Select(entry => entry.Name), Does.Contain("Firmware/dfu/streamed.bin")); + } finally { + File.Delete(temp); + } + } + + private static MemoryStream BuildIpsw(params (string Name, byte[] Data)[] entries) { + var image = new MemoryStream(); + using (var zip = new ZipArchive(image, ZipArchiveMode.Create, leaveOpen: true)) { + foreach (var (name, data) in entries) { + var entry = zip.CreateEntry(name, CompressionLevel.NoCompression); + using var stream = entry.Open(); + stream.Write(data); + } + } + image.Position = 0; + return image; + } + + private static byte[] BuildDataDescriptorIpsw() { + using var backing = new MemoryStream(); + using (var forwardOnly = new ForwardOnlyWriteStream(backing)) + using (var zip = new ZipArchive(forwardOnly, ZipArchiveMode.Create, leaveOpen: true)) { + var entry = zip.CreateEntry("BuildManifest.plist", CompressionLevel.Optimal); + using var stream = entry.Open(); + stream.Write(Enumerable.Repeat((byte)'A', 4096).ToArray()); + } + return backing.ToArray(); + } + + private static string[] RawNames(Stream image) { + image.Position = 0; + using var zip = new ZipArchive(image, ZipArchiveMode.Read, leaveOpen: true); + return zip.Entries.Select(entry => entry.FullName).ToArray(); + } + + private static byte[] ReadEntry(Stream image, string name) { + image.Position = 0; + using var zip = new ZipArchive(image, ZipArchiveMode.Read, leaveOpen: true); + var entry = zip.GetEntry(name) ?? throw new KeyNotFoundException(name); + using var source = entry.Open(); + using var output = new MemoryStream(); + source.CopyTo(output); + return output.ToArray(); + } + + private sealed class ForwardOnlyWriteStream(Stream inner) : Stream { + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => throw new NotSupportedException(); } + public override void Flush() => inner.Flush(); + public override Task FlushAsync(CancellationToken cancellationToken) => inner.FlushAsync(cancellationToken); + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => inner.Write(buffer, offset, count); + public override void Write(ReadOnlySpan buffer) => inner.Write(buffer); + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + => inner.WriteAsync(buffer, cancellationToken); + + protected override void Dispose(bool disposing) { + if (disposing) + inner.Flush(); + base.Dispose(disposing); + } + } +} diff --git a/Compression.Tests/Ipsw/IpswTests.cs b/Compression.Tests/Ipsw/IpswTests.cs index da535ab9e..b6c795774 100644 --- a/Compression.Tests/Ipsw/IpswTests.cs +++ b/Compression.Tests/Ipsw/IpswTests.cs @@ -15,10 +15,10 @@ private static byte[] BuildSyntheticIpsw() { "ProductBuildVersion21E236" + "ProductTypeiPhone15,3" + "")); - WriteEntry(zip, "Firmware/sep-firmware.bin", new byte[] { 0xA, 0xB, 0xC }); - WriteEntry(zip, "Firmware/dfu/iBSS.d22.RELEASE.im4p", new byte[] { 0xD }); - WriteEntry(zip, "LLB.iphone15,3.RELEASE.im4p", new byte[] { 0xE }); - WriteEntry(zip, "iBoot.iphone15,3.RELEASE.im4p", new byte[] { 0xF }); + WriteEntry(zip, "Firmware/sep-firmware.bin", [0xA, 0xB, 0xC]); + WriteEntry(zip, "Firmware/dfu/iBSS.d22.RELEASE.im4p", [0xD]); + WriteEntry(zip, "LLB.iphone15,3.RELEASE.im4p", [0xE]); + WriteEntry(zip, "iBoot.iphone15,3.RELEASE.im4p", [0xF]); WriteEntry(zip, "058-90000-000.dmg", Enumerable.Repeat((byte)0x55, 32).ToArray()); WriteEntry(zip, "Restore.plist", "some plist"u8.ToArray()); } @@ -27,64 +27,71 @@ private static byte[] BuildSyntheticIpsw() { private static void WriteEntry(ZipArchive zip, string name, byte[] data) { var entry = zip.CreateEntry(name, CompressionLevel.NoCompression); - using var es = entry.Open(); - es.Write(data); + using var stream = entry.Open(); + stream.Write(data); } [Test, Category("HappyPath")] public void Descriptor_Properties() { - var d = new FileFormat.Ipsw.IpswFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("Ipsw")); - Assert.That(d.CompoundExtensions, Contains.Item(".ipsw")); - Assert.That(d.Extensions, Is.Empty); - Assert.That(d.MagicSignatures, Is.Empty); - Assert.That(d.Category, Is.EqualTo(Compression.Registry.FormatCategory.Archive)); + var descriptor = new FileFormat.Ipsw.IpswFormatDescriptor(); + Assert.That(descriptor.Id, Is.EqualTo("Ipsw")); + Assert.That(descriptor.CompoundExtensions, Contains.Item(".ipsw")); + Assert.That(descriptor.Extensions, Is.Empty); + Assert.That(descriptor.MagicSignatures, Is.Empty); + Assert.That(descriptor.Category, Is.EqualTo(Compression.Registry.FormatCategory.Archive)); } - [Test, Category("HappyPath")] - public void List_SurfacesCanonicalAppleArtifacts() { - var ipsw = BuildSyntheticIpsw(); - var desc = new FileFormat.Ipsw.IpswFormatDescriptor(); - using var ms = new MemoryStream(ipsw); - var entries = desc.List(ms, null); + [Test, Category("HappyPath"), Category("RoundTrip")] + public void List_PreservesExactZipPaths() { + var descriptor = new FileFormat.Ipsw.IpswFormatDescriptor(); + using var image = new MemoryStream(BuildSyntheticIpsw()); + + var names = descriptor.List(image, null).Select(entry => entry.Name).ToList(); - var names = entries.Select(e => e.Name).ToList(); - Assert.That(names, Contains.Item("FULL.ipsw")); - Assert.That(names, Contains.Item("metadata.ini")); - Assert.That(names, Contains.Item("BuildManifest.plist")); - Assert.That(names, Has.Some.EqualTo("Firmware/sep-firmware.bin")); - Assert.That(names, Has.Some.EqualTo("Firmware/iBSS.d22.RELEASE.im4p")); - Assert.That(names, Contains.Item("LLB.iphone15,3.RELEASE.im4p")); - Assert.That(names, Contains.Item("iBoot.iphone15,3.RELEASE.im4p")); - Assert.That(names, Contains.Item("058-90000-000.dmg")); - Assert.That(names, Has.Some.EqualTo("other/Restore.plist")); + Assert.Multiple(() => { + Assert.That(names, Contains.Item("FULL.ipsw")); + Assert.That(names, Contains.Item("metadata.ini")); + Assert.That(names, Contains.Item("BuildManifest.plist")); + Assert.That(names, Contains.Item("Firmware/sep-firmware.bin")); + Assert.That(names, Contains.Item("Firmware/dfu/iBSS.d22.RELEASE.im4p")); + Assert.That(names, Contains.Item("LLB.iphone15,3.RELEASE.im4p")); + Assert.That(names, Contains.Item("iBoot.iphone15,3.RELEASE.im4p")); + Assert.That(names, Contains.Item("058-90000-000.dmg")); + Assert.That(names, Contains.Item("Restore.plist")); + Assert.That(names, Does.Not.Contain("Firmware/iBSS.d22.RELEASE.im4p")); + Assert.That(names, Does.Not.Contain("other/Restore.plist")); + }); } - [Test, Category("HappyPath")] - public void Extract_WritesCanonicalArtifactsAndMetadata() { - var ipsw = BuildSyntheticIpsw(); - var desc = new FileFormat.Ipsw.IpswFormatDescriptor(); - var tmp = Path.Combine(Path.GetTempPath(), "ipsw_test_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(tmp); + [Test, Category("HappyPath"), Category("RoundTrip")] + public void Extract_PreservesZipPathsAndRendersMetadata() { + var descriptor = new FileFormat.Ipsw.IpswFormatDescriptor(); + var temp = Path.Combine(Path.GetTempPath(), "ipsw_test_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(temp); try { - using var ms = new MemoryStream(ipsw); - desc.Extract(ms, tmp, null, null); + using var image = new MemoryStream(BuildSyntheticIpsw()); + descriptor.Extract(image, temp, null, null); - Assert.That(File.Exists(Path.Combine(tmp, "FULL.ipsw")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "BuildManifest.plist")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "Firmware", "sep-firmware.bin")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "LLB.iphone15,3.RELEASE.im4p")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "iBoot.iphone15,3.RELEASE.im4p")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "058-90000-000.dmg")), Is.True); - Assert.That(File.Exists(Path.Combine(tmp, "other", "Restore.plist")), Is.True); + Assert.Multiple(() => { + Assert.That(File.Exists(Path.Combine(temp, "FULL.ipsw")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "BuildManifest.plist")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "Firmware", "sep-firmware.bin")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "Firmware", "dfu", "iBSS.d22.RELEASE.im4p")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "LLB.iphone15,3.RELEASE.im4p")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "iBoot.iphone15,3.RELEASE.im4p")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "058-90000-000.dmg")), Is.True); + Assert.That(File.Exists(Path.Combine(temp, "Restore.plist")), Is.True); + }); - var meta = File.ReadAllText(Path.Combine(tmp, "metadata.ini")); - Assert.That(meta, Does.Contain("product_version=17.4.1")); - Assert.That(meta, Does.Contain("build_version=21E236")); - Assert.That(meta, Does.Contain("identifier=iPhone15,3")); - Assert.That(meta, Does.Contain("total_zip_entries=7")); + var metadata = File.ReadAllText(Path.Combine(temp, "metadata.ini")); + Assert.Multiple(() => { + Assert.That(metadata, Does.Contain("product_version=17.4.1")); + Assert.That(metadata, Does.Contain("build_version=21E236")); + Assert.That(metadata, Does.Contain("identifier=iPhone15,3")); + Assert.That(metadata, Does.Contain("total_zip_entries=7")); + }); } finally { - Directory.Delete(tmp, recursive: true); + Directory.Delete(temp, recursive: true); } } } diff --git a/Compression.Tests/Lustre/LustreDetectionTests.cs b/Compression.Tests/Lustre/LustreDetectionTests.cs index 7910ce7c2..a2265df85 100644 --- a/Compression.Tests/Lustre/LustreDetectionTests.cs +++ b/Compression.Tests/Lustre/LustreDetectionTests.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Text; using Compression.Registry; using FileSystem.Ext; @@ -27,13 +28,21 @@ public void Descriptor_IdentifiesByMagicAndExtension() { Assert.That(d.MagicSignatures, Has.Count.EqualTo(2)); Assert.That(d.MagicSignatures[0].Bytes, Is.EqualTo("LUSTRE"u8.ToArray())); Assert.That(d.MagicSignatures[1].Bytes, Is.EqualTo(new byte[] { 0x4C, 0x55, 0x73, 0x74 })); - // Promoted R/O — exposes multi-entry + directory capabilities; still no Create/Modify. + + // The namespace projection remains read-only: Lustre LMA/LOV/FID xattrs are opaque, + // so CanModify would over-promise add/remove/purge semantics. Maintenance below that + // layer is still safe where it only touches proven-free blocks / trailing geometry. Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanList), Is.True); Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanExtract), Is.True); Assert.That(d.Capabilities.HasFlag(FormatCapabilities.SupportsDirectories), Is.True); Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); Assert.That(d, Is.Not.InstanceOf()); + Assert.That(d, Is.Not.InstanceOf()); + Assert.That(d, Is.InstanceOf()); + Assert.That(d, Is.InstanceOf()); + Assert.That(d, Is.InstanceOf()); + Assert.That(d, Is.Not.InstanceOf()); } [Test, Category("HappyPath")] @@ -65,30 +74,82 @@ public void NoMagic_NoExt4_Throws() { Assert.Throws(() => d.List(ms, password: null)); } - [Test, Category("Stub")] + [Test, Category("ExceptionalCase")] + public void Legacy_Dump_MaintenanceRefusesNonLdiskfsBytes() { + var d = new LustreFormatDescriptor(); + using var wipe = new MemoryStream(BuildMinimalLegacy(), writable: true); + Assert.Throws(() => ((IWipeEmpty)d).WipeUnusedSpace(wipe)); + + using var source = new MemoryStream(BuildMinimalLegacy(), writable: false); + using var target = new MemoryStream(); + Assert.Throws(() => ((IArchiveShrinkable)d).Shrink(source, target)); + } + + [Test, Category("HappyPath")] public void Description_FlagsLdiskfsDelegationAndOutOfScope() { var d = new LustreFormatDescriptor(); var desc = d.Description.ToLowerInvariant(); Assert.That(desc, Does.Contain("ldiskfs")); Assert.That(desc, Does.Contain("ext4")); Assert.That(desc, Does.Contain("out of scope")); + Assert.That(desc, Does.Contain("shrink")); + Assert.That(desc, Does.Contain("wipe")); } - // ── ldiskfs (ext4) R/O delegation path ────────────────────────────────── + // ── ldiskfs (ext4) delegation + conservative maintenance ──────────────── + + private static byte[] BuildLdiskfsImage(string volumeLabel, params (string Name, byte[] Data)[] files) + => BuildLdiskfsImage(4096, volumeLabel, files); - private static byte[] BuildLdiskfsImage(string volumeLabel, params (string Name, byte[] Data)[] files) { + private static byte[] BuildLdiskfsImage(int totalBlocks, string volumeLabel, params (string Name, byte[] Data)[] files) { var writer = new ExtWriter(); foreach (var (name, data) in files) writer.AddFile(name, data); // ext4 + journal, 4 KB blocks, volume label set (Lustre convention: "lustre-OST0000" / "MGS" / "lustre-MDT0000"). return writer.Build( blockSize: 4096, - totalBlocks: 4096, + totalBlocks: totalBlocks, version: ExtWriter.ExtVersion.Ext4, journal: true, volumeLabel: volumeLabel, inodeSize: 256); } + private static (int BlockSize, long BlockOffset, long DescriptorOffset) FindInitializedFreeBlock(byte[] image) { + var sb = image.AsSpan(1024, 1024); + var blockSize = 1024 << (int)BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(24, 4)); + var firstDataBlock = BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(20, 4)); + var blocksPerGroup = BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(32, 4)); + var featureIncompat = BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(96, 4)); + ulong totalBlocks = BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(4, 4)); + if ((featureIncompat & 0x80) != 0) + totalBlocks |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(sb.Slice(0x150, 4)) << 32; + var descriptorSize = (featureIncompat & 0x80) != 0 + ? Math.Max(32, (int)BinaryPrimitives.ReadUInt16LittleEndian(sb.Slice(0xFE, 2))) + : 32; + var groupCount = (totalBlocks - firstDataBlock + blocksPerGroup - 1) / blocksPerGroup; + var descriptorTableOffset = (long)(firstDataBlock + 1UL) * blockSize; + + for (ulong group = 0; group < groupCount; ++group) { + var descriptorOffset = descriptorTableOffset + (long)group * descriptorSize; + var descriptor = image.AsSpan((int)descriptorOffset, descriptorSize); + var flags = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.Slice(0x12, 2)); + if ((flags & 0x0002) != 0) continue; // BLOCK_UNINIT + + ulong bitmapBlock = BinaryPrimitives.ReadUInt32LittleEndian(descriptor[..4]); + if (descriptorSize >= 64) + bitmapBlock |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(descriptor.Slice(0x20, 4)) << 32; + var bitmapOffset = checked((long)bitmapBlock * blockSize); + var bitmap = image.AsSpan((int)bitmapOffset, blockSize); + var groupFirst = firstDataBlock + group * blocksPerGroup; + var blocksInGroup = (int)Math.Min((ulong)blocksPerGroup, totalBlocks - groupFirst); + for (var bit = 0; bit < blocksInGroup; ++bit) + if ((bitmap[bit >> 3] & (1 << (bit & 7))) == 0) + return (blockSize, checked((long)(groupFirst + (uint)bit) * blockSize), descriptorOffset); + } + + throw new AssertionException("Test image unexpectedly contains no initialized free ldiskfs block."); + } + [Test, Category("HappyPath")] public void Ldiskfs_Image_RecognisedAndDelegated() { var d = new LustreFormatDescriptor(); @@ -108,7 +169,7 @@ public void Ldiskfs_Image_RecognisedAndDelegated() { } [Test, Category("HappyPath")] - public void Ldiskfs_Metadata_DocumentsPartialRoStatus() { + public void Ldiskfs_Metadata_DocumentsPartialBackingStoreStatus() { var d = new LustreFormatDescriptor(); var data = BuildLdiskfsImage("lustre-MDT0000", ("CONFIGS/mountdata", "lustre MDT config"u8.ToArray())); @@ -120,6 +181,7 @@ public void Ldiskfs_Metadata_DocumentsPartialRoStatus() { Assert.That(text, Does.Contain("parse_status=partial-ldiskfs")); Assert.That(text, Does.Contain("backing_fs=ldiskfs (ext4-compatible)")); Assert.That(text, Does.Contain("ldiskfs_volume_label=lustre-MDT0000")); + Assert.That(text, Does.Contain("bitmap-proven free blocks")); Assert.That(text, Does.Contain("not the Lustre logical view").IgnoreCase .Or.Contain("NOT the Lustre logical view")); } @@ -146,6 +208,87 @@ public void Ldiskfs_Extract_ReturnsFileBytesFromDelegatedReader() { "ldiskfs delegation must return the exact file bytes the ext4 reader produces."); } + [Test, Category("HappyPath")] + public void Ldiskfs_Wipe_ZeroesBitmapFreeBlocks_WithoutTouchingLiveObjects() { + var d = new LustreFormatDescriptor(); + var payload = Enumerable.Range(0, 1536).Select(i => (byte)(i * 17)).ToArray(); + var data = BuildLdiskfsImage("lustre-OST0000", ("OBJECTS/0_42", payload)); + var (blockSize, freeOffset, _) = FindInitializedFreeBlock(data); + data.AsSpan((int)freeOffset, blockSize).Fill(0xA5); + + using var image = new MemoryStream(data, writable: true); + var wiped = ((IWipeEmpty)d).WipeUnusedSpace(image, wipeClusterTips: true, wipeDeletedEntries: true); + + Assert.That(wiped, Is.GreaterThanOrEqualTo(blockSize)); + var after = image.ToArray(); + Assert.That(after.AsSpan((int)freeOffset, blockSize).ToArray(), Is.All.EqualTo((byte)0)); + + using var verify = new MemoryStream(after, writable: false); + using var entry = ((IArchiveFormatOperations)d).OpenEntry(verify, "ldiskfs/OBJECTS/0_42", password: null); + using var actual = new MemoryStream(); + entry.CopyTo(actual); + Assert.That(actual.ToArray(), Is.EqualTo(payload), + "Bitmap-driven wipe must not change any allocated object bytes."); + } + + [Test, Category("BoundaryCase")] + public void Ldiskfs_Wipe_FailsClosedForBlockUninitGroup() { + var d = new LustreFormatDescriptor(); + var data = BuildLdiskfsImage("lustre-OST0000", ("OBJECTS/0_1", "live"u8.ToArray())); + var (blockSize, freeOffset, descriptorOffset) = FindInitializedFreeBlock(data); + data.AsSpan((int)freeOffset, blockSize).Fill(0x5A); + + var flags = BinaryPrimitives.ReadUInt16LittleEndian(data.AsSpan((int)descriptorOffset + 0x12, 2)); + BinaryPrimitives.WriteUInt16LittleEndian(data.AsSpan((int)descriptorOffset + 0x12, 2), (ushort)(flags | 0x0002)); + + using var image = new MemoryStream(data, writable: true); + _ = ((IWipeEmpty)d).WipeUnusedSpace(image); + + Assert.That(image.ToArray().AsSpan((int)freeOffset, blockSize).ToArray(), Is.All.EqualTo((byte)0x5A), + "BLOCK_UNINIT means the on-disk bitmap is not authoritative; the whole group must be left alone."); + } + + [Test, Category("HappyPath")] + public void Ldiskfs_Shrink_TrimsTrailingFreeBlocks_AndPreservesTargetIdentityAndPayload() { + var d = new LustreFormatDescriptor(); + var payload = Enumerable.Range(0, 8192).Select(i => (byte)(i ^ (i >> 8))).ToArray(); + var data = BuildLdiskfsImage(8192, "lustre-MDT0000", ("OBJECTS/0_7", payload)); + + using var source = new MemoryStream(data, writable: false); + using var target = new MemoryStream(); + ((IArchiveShrinkable)d).Shrink(source, target); + + Assert.That(target.Length, Is.LessThan(data.LongLength), + "A small target in an oversized one-group ldiskfs image should lose its trailing free blocks."); + + target.Position = 0; + using var meta = ((IArchiveFormatOperations)d).OpenEntry(target, "metadata.ini", password: null); + using var sr = new StreamReader(meta); + Assert.That(sr.ReadToEnd(), Does.Contain("ldiskfs_volume_label=lustre-MDT0000"), + "Shrink must retain the Lustre target label rather than rebuilding as generic ext."); + + target.Position = 0; + using var entry = ((IArchiveFormatOperations)d).OpenEntry(target, "ldiskfs/OBJECTS/0_7", password: null); + using var actual = new MemoryStream(); + entry.CopyTo(actual); + Assert.That(actual.ToArray(), Is.EqualTo(payload)); + } + + [Test, Category("HappyPath")] + public void Ldiskfs_LayoutAnalysis_ReportsGeometryWithoutClaimingStructuralRelayout() { + var d = new LustreFormatDescriptor(); + var data = BuildLdiskfsImage("lustre-OST0000", ("OBJECTS/0_1", "payload"u8.ToArray())); + using var image = new MemoryStream(data, writable: false); + + var analysis = ((ILayoutOptimizable)d).AnalyzeLayout(image); + + Assert.That(analysis.CurrentUnitSize, Is.EqualTo(4096)); + Assert.That(analysis.OptimalUnitSize, Is.EqualTo(4096)); + Assert.That(analysis.PotentialSavingsBytes, Is.Zero); + Assert.That(analysis.RequiresRebuild, Is.Empty); + Assert.That(analysis.Notes.Any(n => n.Contains("not offered", StringComparison.OrdinalIgnoreCase)), Is.True); + } + [Test, Category("BoundaryCase")] public void Ldiskfs_Reader_PreservesRawImageAsLustreObjectBin() { var d = new LustreFormatDescriptor(); diff --git a/Compression.Tests/Lustre/LustreMaintenanceSafetyTests.cs b/Compression.Tests/Lustre/LustreMaintenanceSafetyTests.cs new file mode 100644 index 000000000..a801e606e --- /dev/null +++ b/Compression.Tests/Lustre/LustreMaintenanceSafetyTests.cs @@ -0,0 +1,71 @@ +using System.Buffers.Binary; +using Compression.Registry; +using FileSystem.Ext; +using FileSystem.Lustre; + +namespace Compression.Tests.Lustre; + +[TestFixture] +public sealed class LustreMaintenanceSafetyTests { + + [Test, Category("ExceptionalCase")] + public void Wipe_RefusesCorruptedBlockBitmapBeforeWritingAnything() { + var data = BuildChecksummedLdiskfsImage(); + CorruptFirstBlockBitmap(data); + var expected = (byte[])data.Clone(); + + using var image = new MemoryStream(data, writable: true); + var descriptor = new LustreFormatDescriptor(); + + var ex = Assert.Throws(() => ((IWipeEmpty)descriptor).WipeUnusedSpace(image)); + Assert.That(ex!.Message, Does.Contain("bitmap checksum mismatch").IgnoreCase); + Assert.That(image.ToArray(), Is.EqualTo(expected), + "Checksum failure must happen before the wipe overwrites any block."); + } + + [Test, Category("ExceptionalCase")] + public void Shrink_RefusesCorruptedBlockBitmapWithoutTouchingOutput() { + var data = BuildChecksummedLdiskfsImage(); + CorruptFirstBlockBitmap(data); + + using var source = new MemoryStream(data, writable: false); + using var target = new MemoryStream([0x51, 0x52, 0x53, 0x54], writable: true); + var descriptor = new LustreFormatDescriptor(); + + var ex = Assert.Throws(() => ((IArchiveShrinkable)descriptor).Shrink(source, target)); + Assert.That(ex!.Message, Does.Contain("bitmap checksum mismatch").IgnoreCase); + Assert.That(target.ToArray(), Is.EqualTo(new byte[] { 0x51, 0x52, 0x53, 0x54 }), + "Shrink preflight must validate source allocation metadata before replacing output."); + } + + private static byte[] BuildChecksummedLdiskfsImage() { + var writer = new ExtWriter(); + writer.AddFile("OBJECTS/0_1", Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray()); + return writer.Build( + blockSize: 4096, + totalBlocks: 4096, + version: ExtWriter.ExtVersion.Ext4, + journal: true, + volumeLabel: "lustre-OST0000", + inodeSize: 256); + } + + private static void CorruptFirstBlockBitmap(byte[] image) { + var superblock = image.AsSpan(1024, 1024); + var blockSize = 1024 << (int)BinaryPrimitives.ReadUInt32LittleEndian(superblock.Slice(24, 4)); + var firstDataBlock = BinaryPrimitives.ReadUInt32LittleEndian(superblock.Slice(20, 4)); + var featureIncompat = BinaryPrimitives.ReadUInt32LittleEndian(superblock.Slice(96, 4)); + var descriptorSize = (featureIncompat & 0x0080) != 0 + ? BinaryPrimitives.ReadUInt16LittleEndian(superblock.Slice(0xFE, 2)) + : 32; + var descriptorOffset = checked((int)((firstDataBlock + 1L) * blockSize)); + var descriptor = image.AsSpan(descriptorOffset, descriptorSize); + + ulong bitmapBlock = BinaryPrimitives.ReadUInt32LittleEndian(descriptor[..4]); + if (descriptorSize >= 64) + bitmapBlock |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(descriptor.Slice(0x20, 4)) << 32; + + var bitmapOffset = checked((int)(bitmapBlock * (ulong)blockSize)); + image[bitmapOffset] ^= 0x01; + } +} diff --git a/Compression.Tests/Lynx/LynxTests.cs b/Compression.Tests/Lynx/LynxTests.cs index 455e3e3ff..667bb3c97 100644 --- a/Compression.Tests/Lynx/LynxTests.cs +++ b/Compression.Tests/Lynx/LynxTests.cs @@ -23,6 +23,9 @@ public void Descriptor_CreatesCanonicalMultiFileArchive_AndRoundTripsBytes() { Assert.That(Encoding.ASCII.GetString(bytes, 60, 4), Is.EqualTo("LYNX")); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); }); archive.Position = 0; @@ -103,6 +106,25 @@ public void Descriptor_SupportsEmptyArchive_AndPurgeToZeroEntries() { Assert.That(archive.Length, Is.EqualTo(ReadDirectoryBlockCount(archive.ToArray()) * 254L)); } + [Test] + public void Purge_RebuildsMinimalValidEmptyArchive() { + var descriptor = new LynxFormatDescriptor(); + using var archive = Create(descriptor, [ + ArchiveInputInfo.InMemory("FIRST.PRG", Enumerable.Repeat((byte)0x11, 300).ToArray()), + ArchiveInputInfo.InMemory("SECOND.PRG", Enumerable.Repeat((byte)0x22, 400).ToArray()), + ]); + + descriptor.Purge(archive); + + Assert.Multiple(() => { + Assert.That(archive.Length, Is.EqualTo(254)); + Assert.That(ReadDirectoryBlockCount(archive.ToArray()), Is.EqualTo(1)); + }); + archive.Position = 0; + Assert.That(descriptor.List(archive, null), Is.Empty); + Assert.That(Encoding.ASCII.GetString(archive.ToArray(), 60, 4), Is.EqualTo("LYNX")); + } + [Test] public void Writer_UsesRequestedCommodoreType_AndRejectsLossyNamesOrEncryption() { var descriptor = new LynxFormatDescriptor(); @@ -119,8 +141,8 @@ public void Writer_UsesRequestedCommodoreType_AndRejectsLossyNamesOrEncryption() ArchiveInputInfo.InMemory("abcdefghijklmnop-A", Array.Empty()), ArchiveInputInfo.InMemory("abcdefghijklmnop-B", new byte[] { 1 }), ])); - Assert.Throws(() => Create(descriptor, - [ArchiveInputInfo.InMemory("MÄDCHEN.PRG", Array.Empty())])); + Assert.Throws(() => Create(descriptor, [ + ArchiveInputInfo.InMemory("MÄDCHEN.PRG", Array.Empty())])); Assert.Throws(() => Create(descriptor, [ArchiveInputInfo.InMemory("A.PRG", Array.Empty())], new FormatCreateOptions { Password = "secret" })); @@ -145,6 +167,75 @@ public void Defragment_TrimsOnlyTrailingTransportPadding() { Assert.That(descriptor.ExtractEntryToMemory(archive, "A.PRG", null), Is.EqualTo(Enumerable.Repeat((byte)1, 300).ToArray())); } + [Test] + public void Shrink_ReclaimsDirectoryOverallocationAndTrailer_WithoutChangingPayload() { + var descriptor = new LynxFormatDescriptor(); + var payload = Enumerable.Range(0, 700).Select(index => (byte)(index * 13)).ToArray(); + using var archive = Create(descriptor, [ArchiveInputInfo.InMemory("KEEP.PRG", payload)]); + var transientNames = new List(); + + for (var i = 0; i < 12; ++i) { + var name = $"TEMP{i:D2}.PRG"; + transientNames.Add(name); + descriptor.Add(archive, [ArchiveInputInfo.InMemory(name, Enumerable.Repeat((byte)(i + 1), 20 + i).ToArray())]); + } + + descriptor.Remove(archive, transientNames.ToArray()); + Assert.That(ReadDirectoryBlockCount(archive.ToArray()), Is.GreaterThan(1)); + archive.Position = archive.Length; + archive.Write(Enumerable.Repeat((byte)0xEE, 113).ToArray()); + var bloatedLength = archive.Length; + + using var shrunk = new MemoryStream(); + descriptor.Shrink(archive, shrunk); + + Assert.Multiple(() => { + Assert.That(shrunk.Length, Is.LessThan(bloatedLength)); + Assert.That(shrunk.Length % 254, Is.Zero); + Assert.That(ReadDirectoryBlockCount(shrunk.ToArray()), Is.EqualTo(1)); + }); + shrunk.Position = 0; + Assert.That(descriptor.List(shrunk, null).Select(entry => entry.Name), Is.EqualTo(new[] { "KEEP.PRG" })); + shrunk.Position = 0; + Assert.That(descriptor.ExtractEntryToMemory(shrunk, "KEEP.PRG", null), Is.EqualTo(payload)); + } + + [Test] + public void LayoutAndWipe_ExposeAndClearOnlyUnusedBytes() { + var descriptor = new LynxFormatDescriptor(); + var payload = Enumerable.Range(0, 300).Select(index => (byte)(index * 7 + 3)).ToArray(); + using var archive = Create(descriptor, [ArchiveInputInfo.InMemory("DATA.PRG", payload)]); + archive.Position = archive.Length; + archive.Write(Enumerable.Repeat((byte)0xCC, 37).ToArray()); + + archive.Position = 0; + var free = descriptor.EnumerateLayout(archive) + .Where(extent => extent.Kind == DefragBlockKind.Free) + .ToArray(); + Assert.That(free, Is.Not.Empty); + Assert.That(free.Any(extent => extent.FileName == "Directory padding"), Is.True); + Assert.That(free.Any(extent => extent.FileName == "DATA.PRG block padding"), Is.True); + Assert.That(free.Any(extent => extent.FileName == "Trailing transport data"), Is.True); + + foreach (var extent in free) { + archive.Position = extent.Offset; + archive.Write(Enumerable.Repeat((byte)0xA5, checked((int)extent.Length)).ToArray()); + } + + archive.Position = 0; + var wiped = descriptor.WipeUnusedSpace(archive); + Assert.That(wiped, Is.EqualTo(free.Sum(extent => extent.Length))); + + archive.Position = 0; + Assert.That(descriptor.ExtractEntryToMemory(archive, "DATA.PRG", null), Is.EqualTo(payload)); + var bytes = archive.ToArray(); + foreach (var extent in free) + Assert.That( + bytes.AsSpan(checked((int)extent.Offset), checked((int)extent.Length)).ToArray().All(value => value == 0), + Is.True, + $"Expected free extent '{extent.FileName}' to be zeroed."); + } + private static MemoryStream Create( LynxFormatDescriptor descriptor, IReadOnlyList inputs, diff --git a/Compression.Tests/MooseFs/MooseFsTests.cs b/Compression.Tests/MooseFs/MooseFsTests.cs index 153aec269..4cc821bdd 100644 --- a/Compression.Tests/MooseFs/MooseFsTests.cs +++ b/Compression.Tests/MooseFs/MooseFsTests.cs @@ -6,347 +6,411 @@ namespace Compression.Tests.MooseFs; /// -/// Tests for the MooseFS master-metadata partial R/O reader. -/// MooseFS itself is a distributed FS and we don't have golden cluster -/// images, so these tests use synthetic MooseFS-shaped images built byte by -/// byte against the documented envelope: 8-byte signature, 16-byte counters, -/// stream of (8-byte tag, 8-byte BE length, payload) sections, 16-byte EOF -/// marker. This validates the envelope walker without making any claim -/// about the version-specific NODE / EDGE / CHNK record bodies. +/// Tests for the MooseFS master-metadata envelope and the maintenance operations +/// that are safe on a standalone metadata.mfs image. Synthetic images contain +/// only specification-defined framing; NODE/EDGE/CHNK payloads remain opaque. /// [TestFixture] public class MooseFsTests { - private static byte[] BuildHeader(string signature = "MFSM 2.0", - ulong fileIdCounter = 100, - ulong metadataVersion = 42) { + private static byte[] BuildModernHeader( + string signature = "MFSM 2.0", + ulong metadataVersion = 42, + ulong metaId = 100) { if (signature.Length != 8) throw new ArgumentException("MooseFS signature must be exactly 8 ASCII bytes.", nameof(signature)); - var buf = new byte[8 + 16]; - Encoding.ASCII.GetBytes(signature).CopyTo(buf.AsSpan(0, 8)); - BinaryPrimitives.WriteUInt64BigEndian(buf.AsSpan(8, 8), fileIdCounter); - BinaryPrimitives.WriteUInt64BigEndian(buf.AsSpan(16, 8), metadataVersion); - return buf; + + var result = new byte[24]; + Encoding.ASCII.GetBytes(signature).CopyTo(result.AsSpan(0, 8)); + BinaryPrimitives.WriteUInt64BigEndian(result.AsSpan(8, 8), metadataVersion); + BinaryPrimitives.WriteUInt64BigEndian(result.AsSpan(16, 8), metaId); + return result; + } + + private static byte[] BuildPre20Header( + string signature = "MFSM 1.9", + uint maxNodeId = 123, + ulong metadataVersion = 42, + uint nextSessionId = 7) { + if (signature.Length != 8) + throw new ArgumentException("MooseFS signature must be exactly 8 ASCII bytes.", nameof(signature)); + + var result = new byte[24]; + Encoding.ASCII.GetBytes(signature).CopyTo(result.AsSpan(0, 8)); + BinaryPrimitives.WriteUInt32BigEndian(result.AsSpan(8, 4), maxNodeId); + BinaryPrimitives.WriteUInt64BigEndian(result.AsSpan(12, 8), metadataVersion); + BinaryPrimitives.WriteUInt32BigEndian(result.AsSpan(20, 4), nextSessionId); + return result; } private static byte[] BuildSection(string tag, byte[] payload) { if (tag.Length != 8) throw new ArgumentException("MooseFS section tag must be exactly 8 ASCII bytes.", nameof(tag)); - var buf = new byte[8 + 8 + payload.Length]; - Encoding.ASCII.GetBytes(tag).CopyTo(buf.AsSpan(0, 8)); - BinaryPrimitives.WriteUInt64BigEndian(buf.AsSpan(8, 8), (ulong)payload.Length); - payload.CopyTo(buf.AsSpan(16)); - return buf; + + var result = new byte[16 + payload.Length]; + Encoding.ASCII.GetBytes(tag).CopyTo(result.AsSpan(0, 8)); + BinaryPrimitives.WriteUInt64BigEndian(result.AsSpan(8, 8), (ulong)payload.Length); + payload.CopyTo(result.AsSpan(16)); + return result; } private static byte[] EofMarker() => "[MFS EOF MARKER]"u8.ToArray(); private static byte[] Concat(params byte[][] parts) { - var total = parts.Sum(p => p.Length); - var buf = new byte[total]; - var off = 0; - foreach (var p in parts) { - p.CopyTo(buf.AsSpan(off)); - off += p.Length; + var total = parts.Sum(static p => p.Length); + var result = new byte[total]; + var offset = 0; + foreach (var part in parts) { + part.CopyTo(result.AsSpan(offset)); + offset += part.Length; } - return buf; + return result; } - private static byte[] BuildMinimalImageWithSections(params (string Tag, int PayloadLen)[] sections) { - var parts = new List { BuildHeader() }; - foreach (var (tag, len) in sections) { - var payload = new byte[len]; - for (var i = 0; i < len; i++) payload[i] = (byte)(i & 0xFF); + private static byte[] BuildModernImage(params (string Tag, int PayloadLength)[] sections) { + var parts = new List { BuildModernHeader() }; + foreach (var (tag, length) in sections) { + var payload = new byte[length]; + for (var i = 0; i < payload.Length; ++i) + payload[i] = (byte)i; parts.Add(BuildSection(tag, payload)); } parts.Add(EofMarker()); return Concat(parts.ToArray()); } - // ============================================================ - // Descriptor sanity checks - // ============================================================ - [Test, Category("HappyPath")] public void Descriptor_IdentifiesByMagic() { - var d = new MooseFsFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("MooseFs")); - Assert.That(d.Extensions, Does.Contain(".mfsm")); - Assert.That(d.Extensions, Does.Not.Contain(".mfs"), - "MooseFS must not claim .mfs — it collides with the Macintosh File System (FileSystem.Mfs)."); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(1)); - Assert.That(d.MagicSignatures[0].Bytes, Is.EqualTo("MFSM"u8.ToArray())); + var descriptor = new MooseFsFormatDescriptor(); + Assert.That(descriptor.Id, Is.EqualTo("MooseFs")); + Assert.That(descriptor.Extensions, Does.Contain(".mfsm")); + Assert.That(descriptor.Extensions, Does.Not.Contain(".mfs"), + "MooseFS must not claim .mfs because it collides with Macintosh File System."); + Assert.That(descriptor.MagicSignatures, Has.Count.EqualTo(1)); + Assert.That(descriptor.MagicSignatures[0].Bytes, Is.EqualTo("MFSM"u8.ToArray())); } [Test, Category("HappyPath")] - public void Descriptor_DoesNotAdvertiseCreateOrModify() { - var d = new MooseFsFormatDescriptor(); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False, - "MooseFS metadata.mfs is not writable from a single image — chunks live on chunk servers."); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanList), Is.True); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanExtract), Is.True); + public void Descriptor_AdvertisesOnlyHonestStandaloneCapabilities() { + var descriptor = new MooseFsFormatDescriptor(); + Assert.Multiple(() => { + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanList), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanExtract), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + }); } [Test, Category("HappyPath")] public void Descriptor_DescriptionStatesChunkServerLimitation() { - var d = new MooseFsFormatDescriptor(); - var desc = d.Description.ToLowerInvariant(); - Assert.That(desc, Does.Contain("chunk server").Or.Contain("chunk-server").Or.Contain("chunkserver"), - "Description must be honest about file data living on chunk servers."); + var description = new MooseFsFormatDescriptor().Description.ToLowerInvariant(); + Assert.That(description, Does.Contain("chunk server")); + Assert.That(description, Does.Contain("not claimed")); } - // ============================================================ - // Reader — equivalence classes for the parser entry - // ============================================================ - [Test, Category("Boundary")] - public void Reader_RejectsImageSmallerThanHeader() { - using var ms = new MemoryStream(new byte[] { 0x4D, 0x46, 0x53 }); // "MFS", missing M - Assert.Throws(() => _ = new MooseFsReader(ms)); + public void Reader_RejectsImageSmallerThanSignature() { + using var stream = new MemoryStream("MFSM"u8.ToArray()); + Assert.Throws(() => _ = new MooseFsReader(stream)); } [Test, Category("Exception")] public void Reader_RejectsWrongMagic() { - var buf = new byte[64]; - Encoding.ASCII.GetBytes("NOPENOPE").CopyTo(buf.AsSpan(0, 8)); - using var ms = new MemoryStream(buf); - Assert.Throws(() => _ = new MooseFsReader(ms)); + using var stream = new MemoryStream("NOPENOPE"u8.ToArray()); + Assert.Throws(() => _ = new MooseFsReader(stream)); } [Test, Category("HappyPath")] - public void Reader_HeaderOnly_ParsesSignatureAndCounters() { - // Just header (8 bytes signature + 16 bytes counters) + EOF marker — - // no sections. ParseStatus must be "ok" since the EOF marker IS the - // legal end of the stream. - var image = Concat(BuildHeader(), EofMarker()); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.Signature, Is.EqualTo("MFSM 2.0")); - Assert.That(r.FileIdCounter, Is.EqualTo(100UL)); - Assert.That(r.MetadataVersion, Is.EqualTo(42UL)); - Assert.That(r.ParseStatus, Is.EqualTo("ok")); - Assert.That(r.Sections, Is.Empty); - Assert.That(r.Entries.Select(e => e.Name), - Is.EquivalentTo(new[] { "metadata.ini", "moosefs-master.bin" })); + public void Reader_ModernHeader_ParsesMetadataVersionAndMetaId() { + var image = Concat(BuildModernHeader(metadataVersion: 42, metaId: 100), EofMarker()); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.Multiple(() => { + Assert.That(reader.ValidHeader, Is.True); + Assert.That(reader.Signature, Is.EqualTo("MFSM 2.0")); + Assert.That(reader.FileFormatVersion, Is.EqualTo(0x20)); + Assert.That(reader.MetadataVersion, Is.EqualTo(42UL)); + Assert.That(reader.MetaId, Is.EqualTo(100UL)); + Assert.That(reader.MaxNodeId, Is.Null); + Assert.That(reader.NextSessionId, Is.Null); + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + Assert.That(reader.Sections, Is.Empty); + }); } [Test, Category("Boundary")] - public void Reader_TooShortForCounters_StaysHeaderOnly() { - // Only signature, no room for 16-byte counters or sections. - var image = "MFSM 2.0"u8.ToArray(); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.Signature, Is.EqualTo("MFSM 2.0")); - Assert.That(r.FileIdCounter, Is.Null); - Assert.That(r.MetadataVersion, Is.Null); - Assert.That(r.ParseStatus, Is.EqualTo("header-only")); - } - - // ============================================================ - // Reader — section index walk - // ============================================================ + public void Reader_TooShortForMetadataHeader_StaysHeaderOnly() { + using var stream = new MemoryStream("MFSM 2.0"u8.ToArray()); + using var reader = new MooseFsReader(stream); + Assert.Multiple(() => { + Assert.That(reader.FileFormatVersion, Is.EqualTo(0x20)); + Assert.That(reader.MetadataVersion, Is.Null); + Assert.That(reader.MetaId, Is.Null); + Assert.That(reader.ParseStatus, Is.EqualTo("header-only")); + }); + } [Test, Category("HappyPath")] - public void Reader_WalksAllSections_AndStopsAtEofMarker() { - var image = BuildMinimalImageWithSections( + public void Reader_MfsmNew_IsOfficialEmptyBootstrap() { + using var stream = new MemoryStream("MFSM NEW"u8.ToArray()); + using var reader = new MooseFsReader(stream); + Assert.Multiple(() => { + Assert.That(reader.ValidHeader, Is.True); + Assert.That(reader.IsEmptyBootstrap, Is.True); + Assert.That(reader.FileFormatVersion, Is.Null); + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + Assert.That(reader.Sections, Is.Empty); + }); + } + + [Test, Category("Exception")] + public void Reader_MfsmNewWithTrailingData_IsRejectedAsCanonicalImage() { + var image = Concat("MFSM NEW"u8.ToArray(), [0x42]); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + Assert.That(reader.ParseStatus, Is.EqualTo("trailing-data")); + } + + [Test, Category("HappyPath")] + public void Reader_WalksModernSections_AndStopsAtEofMarker() { + var image = BuildModernImage( ("SESS 1.0", 32), - ("STAT 1.0", 16), ("NODE 1.0", 256), ("EDGE 1.0", 128), ("CHNK 1.0", 64)); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + Assert.That(reader.Sections.Select(static s => s.Tag), + Is.EqualTo(new[] { "SESS 1.0", "NODE 1.0", "EDGE 1.0", "CHNK 1.0" })); + Assert.That(reader.Sections.Select(static s => s.Length), + Is.EqualTo(new long[] { 32, 256, 128, 64 })); + } + + [Test, Category("HappyPath")] + public void Reader_SectionedPre20Format_UsesLegacyHeaderShape() { + var image = Concat( + BuildPre20Header("MFSM 1.9", maxNodeId: 321, metadataVersion: 88, nextSessionId: 9), + BuildSection("NODE 1.0", new byte[16]), + EofMarker()); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.Multiple(() => { + Assert.That(reader.FileFormatVersion, Is.EqualTo(0x19)); + Assert.That(reader.MaxNodeId, Is.EqualTo(321U)); + Assert.That(reader.MetadataVersion, Is.EqualTo(88UL)); + Assert.That(reader.NextSessionId, Is.EqualTo(9U)); + Assert.That(reader.MetaId, Is.Null); + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + Assert.That(reader.Sections, Has.Count.EqualTo(1)); + }); + } - Assert.That(r.ParseStatus, Is.EqualTo("ok")); - Assert.That(r.Sections, Has.Count.EqualTo(5)); - Assert.That(r.Sections.Select(s => s.Tag), - Is.EqualTo(new[] { "SESS 1.0", "STAT 1.0", "NODE 1.0", "EDGE 1.0", "CHNK 1.0" })); - Assert.That(r.Sections.Select(s => s.Length), - Is.EqualTo(new long[] { 32, 16, 256, 128, 64 })); + [Test, Category("HappyPath")] + public void Reader_Pre16Format_RemainsOpaqueAndUsesZeroEofMarker() { + var opaqueLegacyBody = new byte[] { 0x01, 0xA5, 0x7F, 0x00, 0xCC }; + var image = Concat( + BuildPre20Header("MFSM 1.5", maxNodeId: 456, metadataVersion: 99, nextSessionId: 11), + opaqueLegacyBody, + new byte[16]); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.Multiple(() => { + Assert.That(reader.FileFormatVersion, Is.EqualTo(0x15)); + Assert.That(reader.MaxNodeId, Is.EqualTo(456U)); + Assert.That(reader.MetadataVersion, Is.EqualTo(99UL)); + Assert.That(reader.NextSessionId, Is.EqualTo(11U)); + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + Assert.That(reader.Sections, Is.Empty, + "Pre-1.6 bodies are not section-framed and must not be guessed from payload bytes."); + }); } [Test, Category("HappyPath")] - public void Reader_SurfacesPerSectionPayloadAsEntry() { - var image = BuildMinimalImageWithSections( - ("NODE 1.0", 100), - ("EDGE 1.0", 50)); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - - var names = r.Entries.Select(e => e.Name).ToList(); + public void Reader_SurfacesPerSectionPayloadAsSyntheticEntry() { + var image = BuildModernImage(("NODE 1.0", 100), ("EDGE 1.0", 50)); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + var names = reader.Entries.Select(static e => e.Name).ToArray(); Assert.That(names, Does.Contain("metadata.ini")); Assert.That(names, Does.Contain("moosefs-master.bin")); Assert.That(names, Does.Contain("section_NODE_1_0.bin")); Assert.That(names, Does.Contain("section_EDGE_1_0.bin")); - var node = r.Entries.First(e => e.Name == "section_NODE_1_0.bin"); - Assert.That(node.Size, Is.EqualTo(100)); - // First payload byte was set to (byte)(i & 0xFF) = 0 at i=0. - Assert.That(node.Data[0], Is.EqualTo(0)); + var node = reader.Entries.First(static e => e.Name == "section_NODE_1_0.bin"); Assert.That(node.Data[99], Is.EqualTo(99)); } [Test, Category("Exception")] public void Reader_TruncatedSectionLength_MarksTruncated() { - // Build header + a section that claims more bytes than the image actually has. - var header = BuildHeader(); - var tag = "NODE 1.0"u8.ToArray(); - var lenBuf = new byte[8]; - BinaryPrimitives.WriteUInt64BigEndian(lenBuf, 10_000UL); // claim 10k bytes - var shortPayload = new byte[100]; // only 100 actually present - var image = Concat(header, tag, lenBuf, shortPayload); - - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.ParseStatus, Is.EqualTo("truncated")); - Assert.That(r.Sections, Is.Empty, - "A length-overflow section must not be added to the section list."); + var length = new byte[8]; + BinaryPrimitives.WriteUInt64BigEndian(length, 10_000UL); + var image = Concat(BuildModernHeader(), "NODE 1.0"u8.ToArray(), length, new byte[100]); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.That(reader.ParseStatus, Is.EqualTo("truncated")); + Assert.That(reader.Sections, Is.Empty); } [Test, Category("Exception")] public void Reader_NoEofMarker_MarksTruncated() { - // Valid sections but no [MFS EOF MARKER] trailing. - var header = BuildHeader(); - var sec = BuildSection("NODE 1.0", new byte[64]); - var image = Concat(header, sec); // no EOF marker - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ParseStatus, Is.EqualTo("truncated")); - Assert.That(r.Sections, Has.Count.EqualTo(1), - "The walker should still surface every successfully parsed section before noticing the missing EOF marker."); + var image = Concat(BuildModernHeader(), BuildSection("NODE 1.0", new byte[64])); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.That(reader.ParseStatus, Is.EqualTo("truncated")); + Assert.That(reader.Sections, Has.Count.EqualTo(1)); } [Test, Category("Exception")] - public void Reader_NonAsciiSectionTag_MarksTruncated() { - // Image with a "section tag" that's actually binary garbage — walker - // must reject it rather than accepting random bytes as a tag. - var header = BuildHeader(); - var garbageTag = new byte[] { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 }; - var lenBuf = new byte[8]; - BinaryPrimitives.WriteUInt64BigEndian(lenBuf, 0UL); - var image = Concat(header, garbageTag, lenBuf, EofMarker()); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ParseStatus, Is.EqualTo("truncated")); - Assert.That(r.Sections, Is.Empty); - } - - // ============================================================ - // metadata.ini content - // ============================================================ + public void Reader_EofMarkerMustBeLastBytes() { + var image = Concat(BuildModernHeader(), EofMarker(), [0xDE, 0xAD]); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + Assert.That(reader.ParseStatus, Is.EqualTo("trailing-data")); + } - [Test, Category("HappyPath")] - public void MetadataIni_ReportsHeaderAndSectionTable() { - var image = BuildMinimalImageWithSections( - ("NODE 1.0", 100), - ("EDGE 1.0", 50)); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); + [Test, Category("Exception")] + public void Reader_InvalidSectionTag_MarksTruncated() { + var invalidTag = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 }; + var image = Concat(BuildModernHeader(), invalidTag, new byte[8], EofMarker()); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + + Assert.That(reader.ParseStatus, Is.EqualTo("truncated")); + Assert.That(reader.Sections, Is.Empty); + } - var ini = r.Entries.First(e => e.Name == "metadata.ini"); - var text = Encoding.UTF8.GetString(ini.Data); + [Test, Category("HappyPath")] + public void MetadataIni_ReportsCorrectVersionedHeaderAndSectionTable() { + var image = BuildModernImage(("NODE 1.0", 100), ("EDGE 1.0", 50)); + using var stream = new MemoryStream(image); + using var reader = new MooseFsReader(stream); + var text = Encoding.UTF8.GetString(reader.Entries.First(static e => e.Name == "metadata.ini").Data); Assert.That(text, Does.Contain("parse_status=ok")); Assert.That(text, Does.Contain("signature=MFSM 2.0")); - Assert.That(text, Does.Contain("magic_tag=MFSM")); - Assert.That(text, Does.Contain("file_id_counter=100")); + Assert.That(text, Does.Contain("file_format_version=2.0")); Assert.That(text, Does.Contain("metadata_version=42")); + Assert.That(text, Does.Contain("meta_id=100")); + Assert.That(text, Does.Not.Contain("file_id_counter")); Assert.That(text, Does.Contain("section_count=2")); Assert.That(text, Does.Contain("NODE 1.0")); - Assert.That(text, Does.Contain("EDGE 1.0")); - Assert.That(text.ToLowerInvariant(), Does.Contain("chunk server")); + Assert.That(text.ToLowerInvariant(), Does.Contain("chunk servers")); } - [Test, Category("HappyPath")] - public void MetadataIni_HeaderOnly_FlagsHeaderOnlyStatus() { - // Bare 8-byte signature, no counters or sections at all. - var image = "MFSM 2.0"u8.ToArray(); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - var ini = r.Entries.First(e => e.Name == "metadata.ini"); - var text = Encoding.UTF8.GetString(ini.Data); - Assert.That(text, Does.Contain("parse_status=header-only")); - Assert.That(text, Does.Contain("section_count=0")); - } - - // ============================================================ - // Descriptor.List / Extract / OpenEntry — public surface - // ============================================================ - [Test, Category("HappyPath")] public void Descriptor_List_ReturnsMetadataPlusSections() { - var d = new MooseFsFormatDescriptor(); - var image = BuildMinimalImageWithSections(("NODE 1.0", 32)); - using var ms = new MemoryStream(image); - var entries = d.List(ms, password: null); - var names = entries.Select(e => e.Name).ToList(); + var descriptor = new MooseFsFormatDescriptor(); + using var stream = new MemoryStream(BuildModernImage(("NODE 1.0", 32))); + var names = descriptor.List(stream, password: null).Select(static e => e.Name).ToArray(); + Assert.That(names, Does.Contain("metadata.ini")); Assert.That(names, Does.Contain("moosefs-master.bin")); Assert.That(names, Does.Contain("section_NODE_1_0.bin")); } [Test, Category("HappyPath")] - public void Descriptor_Extract_WritesAllSurfacedEntries() { - var d = new MooseFsFormatDescriptor(); - var image = BuildMinimalImageWithSections(("NODE 1.0", 32)); - var tempDir = Path.Combine(Path.GetTempPath(), "moosefs-extract-" + Guid.NewGuid().ToString("N")[..8]); - Directory.CreateDirectory(tempDir); - try { - using var ms = new MemoryStream(image); - d.Extract(ms, tempDir, password: null, files: null); - - Assert.That(File.Exists(Path.Combine(tempDir, "metadata.ini")), Is.True); - Assert.That(File.Exists(Path.Combine(tempDir, "moosefs-master.bin")), Is.True); - Assert.That(File.Exists(Path.Combine(tempDir, "section_NODE_1_0.bin")), Is.True); - } finally { - try { Directory.Delete(tempDir, recursive: true); } catch { /* best effort */ } - } + public void Descriptor_OpenEntry_ReturnsExactlySectionPayload() { + var operations = (IArchiveFormatOperations)new MooseFsFormatDescriptor(); + using var archive = new MemoryStream(BuildModernImage(("NODE 1.0", 100))); + using var entry = operations.OpenEntry(archive, "section_NODE_1_0.bin", password: null); + + Assert.That(entry.Length, Is.EqualTo(100)); + var data = new byte[200]; + Assert.That(entry.Read(data, 0, data.Length), Is.EqualTo(100)); + } + + [Test, Category("Exception")] + public void Descriptor_OpenEntry_UnknownName_Throws() { + var operations = (IArchiveFormatOperations)new MooseFsFormatDescriptor(); + using var archive = new MemoryStream(BuildModernImage(("NODE 1.0", 32))); + Assert.Throws(() => operations.OpenEntry(archive, "does-not-exist.bin", null)); } [Test, Category("HappyPath")] - public void Descriptor_OpenEntry_ReturnsBoundedStream() { - var d = (IArchiveFormatOperations)new MooseFsFormatDescriptor(); - var image = BuildMinimalImageWithSections(("NODE 1.0", 100)); - using var ms = new MemoryStream(image); - using var s = d.OpenEntry(ms, "section_NODE_1_0.bin", password: null); - Assert.That(s.Length, Is.EqualTo(100)); - var buf = new byte[200]; - var n = s.Read(buf, 0, buf.Length); - Assert.That(n, Is.EqualTo(100), - "Bounded entry stream must yield exactly the section payload, never adjacent bytes."); + public void Descriptor_Layout_CoversEveryByteWithoutInventingFreeSpace() { + var image = BuildModernImage(("NODE 1.0", 32), ("EDGE 1.0", 17)); + var layout = (IArchiveLayoutMap)new MooseFsFormatDescriptor(); + using var archive = new MemoryStream(image); + var blocks = layout.EnumerateLayout(archive).ToArray(); + + Assert.Multiple(() => { + Assert.That(blocks, Is.Not.Empty); + Assert.That(blocks.Sum(static block => block.Length), Is.EqualTo(image.LongLength)); + Assert.That(blocks.Any(static block => block.Kind == DefragBlockKind.Free), Is.False); + Assert.That(blocks[0].Offset, Is.Zero); + Assert.That(blocks[^1].Offset + blocks[^1].Length, Is.EqualTo(image.LongLength)); + }); + } + + [Test, Category("HappyPath")] + public void Descriptor_LegacyLayout_IsConservativelyAllMetadata() { + var image = Concat(BuildPre20Header("MFSM 1.5"), [0x01, 0x02, 0x03], new byte[16]); + var layout = (IArchiveLayoutMap)new MooseFsFormatDescriptor(); + using var archive = new MemoryStream(image); + var blocks = layout.EnumerateLayout(archive).ToArray(); + + Assert.That(blocks, Has.Length.EqualTo(1)); + Assert.That(blocks[0].Offset, Is.Zero); + Assert.That(blocks[0].Length, Is.EqualTo(image.LongLength)); + Assert.That(blocks[0].Kind, Is.EqualTo(DefragBlockKind.MetadataReserved)); } [Test, Category("Exception")] - public void Descriptor_OpenEntry_UnknownName_Throws() { - var d = (IArchiveFormatOperations)new MooseFsFormatDescriptor(); - var image = BuildMinimalImageWithSections(("NODE 1.0", 32)); - using var ms = new MemoryStream(image); - Assert.Throws(() => d.OpenEntry(ms, "does-not-exist.bin", password: null)); + public void Descriptor_Layout_FailsClosedForTruncatedImage() { + var image = Concat(BuildModernHeader(), BuildSection("NODE 1.0", new byte[8])); + var layout = (IArchiveLayoutMap)new MooseFsFormatDescriptor(); + using var archive = new MemoryStream(image); + Assert.That(layout.EnumerateLayout(archive), Is.Empty); } - // ============================================================ - // Pre-1.6 signature path (no counters) - // ============================================================ + [Test, Category("HappyPath")] + public void Descriptor_Wipe_IsNoOpForPackedMetadata() { + var original = BuildModernImage(("NODE 1.0", 64), ("EDGE 1.0", 23)); + using var archive = new MemoryStream(); + archive.Write(original); + archive.Position = 0; + + var wiped = ((IWipeEmpty)new MooseFsFormatDescriptor()).WipeUnusedSpace(archive); + + Assert.That(wiped, Is.Zero); + Assert.That(archive.ToArray(), Is.EqualTo(original)); + } [Test, Category("HappyPath")] - public void Reader_LegacyMfsm15Signature_SkipsCounterBlock() { - // For "MFSM 1.4" / "MFSM 1.5", section stream starts immediately after - // the 8-byte signature — there are no counter bytes. We build a minimal - // image and confirm the walker picks up sections from offset 8. - var sig = "MFSM 1.5"u8.ToArray(); - var sec = BuildSection("NODE 1.0", new byte[16]); - var image = Concat(sig, sec, EofMarker()); - using var ms = new MemoryStream(image); - var r = new MooseFsReader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.Signature, Is.EqualTo("MFSM 1.5")); - Assert.That(r.ParseStatus, Is.EqualTo("ok")); - Assert.That(r.Sections, Has.Count.EqualTo(1)); - Assert.That(r.Sections[0].Tag, Is.EqualTo("NODE 1.0")); - Assert.That(r.Sections[0].Length, Is.EqualTo(16)); + public void Descriptor_Purge_ResetsToOfficialEmptyBootstrap() { + using var archive = new MemoryStream(); + archive.Write(BuildModernImage(("NODE 1.0", 64), ("EDGE 1.0", 23))); + archive.Position = 0; + + ((IArchivePurgeable)new MooseFsFormatDescriptor()).Purge(archive); + + Assert.That(archive.ToArray(), Is.EqualTo("MFSM NEW"u8.ToArray())); + archive.Position = 0; + using var reader = new MooseFsReader(archive); + Assert.That(reader.IsEmptyBootstrap, Is.True); + Assert.That(reader.ParseStatus, Is.EqualTo("ok")); + } + + [Test, Category("Exception")] + public void Descriptor_Purge_RejectsStructurallyInvalidMetadata() { + using var archive = new MemoryStream(); + archive.Write(Concat(BuildModernHeader(), BuildSection("NODE 1.0", new byte[8]))); + archive.Position = 0; + + Assert.Throws(() => + ((IArchivePurgeable)new MooseFsFormatDescriptor()).Purge(archive)); } } diff --git a/Compression.Tests/Nwfs386/Nwfs386DetectionTests.cs b/Compression.Tests/Nwfs386/Nwfs386DetectionTests.cs index 4a4e0f0e7..79250ed9b 100644 --- a/Compression.Tests/Nwfs386/Nwfs386DetectionTests.cs +++ b/Compression.Tests/Nwfs386/Nwfs386DetectionTests.cs @@ -1,23 +1,32 @@ #pragma warning disable CS1591 using Compression.Lib; +using FileSystem.Nwfs; namespace Compression.Tests.Nwfs386; [TestFixture] public class Nwfs386DetectionTests { - private static byte[] BuildMinimal() { + [Test, Category("ErrorHandling")] + public void Detector_DoesNotTreatArbitraryNetWPrefixAsNwfs386() { var image = new byte[4096]; - // "NetW" ASCII at offset 0. - image[0] = 0x4E; image[1] = 0x65; image[2] = 0x74; image[3] = 0x57; - return image; + "NetW"u8.CopyTo(image); + + var fmt = FormatDetector.DetectByMagic(image); + + Assert.That(fmt.ToString(), Is.Not.EqualTo("Nwfs386").IgnoreCase, + "The old four-byte NetW prefix had no authoritative NWFS386 basis and must not claim the format."); } [Test, Category("HappyPath")] - public void Detector_IdentifiesNwfs386_ByMagic() { - var image = BuildMinimal(); - var fmt = FormatDetector.DetectByMagic(image.AsSpan(0, 512)); - Assert.That(fmt.ToString(), Is.EqualTo("Nwfs386").IgnoreCase, - $"FormatDetector must recognise NWFS386 via 'NetW' at offset 0. Got: {fmt}"); + public void HotfixMagic_IsOwnedBySharedNwfsDetector_NotDuplicatedByAlias() { + var writer = new NwfsWriter(); + writer.AddFile("HELLO.TXT", "hello"u8.ToArray()); + var image = writer.Build(); + + var fmt = FormatDetector.DetectByMagic(image); + + Assert.That(fmt.ToString(), Is.EqualTo("Nwfs").IgnoreCase, + "Nwfs386 is an extension-routed compatibility id; the shared NWFS descriptor owns HOTFIX00 magic routing."); } } diff --git a/Compression.Tests/Nwfs386/Nwfs386StubBehaviorTests.cs b/Compression.Tests/Nwfs386/Nwfs386StubBehaviorTests.cs index 30385909d..f4ef6e567 100644 --- a/Compression.Tests/Nwfs386/Nwfs386StubBehaviorTests.cs +++ b/Compression.Tests/Nwfs386/Nwfs386StubBehaviorTests.cs @@ -1,64 +1,173 @@ #pragma warning disable CS1591 +using System.Buffers.Binary; using Compression.Registry; +using FileSystem.Nwfs; using FileSystem.Nwfs386; namespace Compression.Tests.Nwfs386; -/// -/// Pins the stub-tier surface for . Novell -/// NetWare 386 has no public on-disk spec — these tests prevent silent capability -/// creep (CanCreate/CanModify) and stop the opaque-blob entry shape from drifting. -/// [TestFixture] -public class Nwfs386StubBehaviorTests { +public class Nwfs386BehaviorTests { - private static byte[] BuildMagicOnly() { - var image = new byte[4096]; - // "NetW" - image[0] = 0x4E; image[1] = 0x65; image[2] = 0x74; image[3] = 0x57; + private static readonly byte[] Hello = "hello nwfs386"u8.ToArray(); + private static readonly byte[] Readme = Enumerable.Range(0, 5000).Select(i => (byte)(i * 31)).ToArray(); + + private static MemoryStream CreateImage(int blockSize = 4096) { + var image = new MemoryStream(); + new Nwfs386FormatDescriptor().Create( + image, + [ + ArchiveInputInfo.InMemory("HELLO.TXT", Hello), + ArchiveInputInfo.InMemory("PUBLIC/README.DOC", Readme), + ], + new FormatCreateOptions { + FormatSpecific = new Dictionary(StringComparer.OrdinalIgnoreCase) { + ["BlockSize"] = blockSize.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["VolumeName"] = "SYS", + }, + }); + image.Position = 0; return image; } - [Test, Category("Stub")] - public void Stub_DescriptorHonestlyAdvertisesCapabilities_AndOpaqueEntries() { - var d = new Nwfs386FormatDescriptor(); - - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False, - "NWFS386 is stub-tier (no public spec) — must not advertise CanCreate."); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False, - "NWFS386 is stub-tier (no public spec) — must not advertise CanModify."); - - var image = BuildMagicOnly(); - using var ms = new MemoryStream(image, writable: false); - var entries = d.List(ms, null); - - var names = entries.Select(e => e.Name).ToList(); - Assert.That(names, Is.EquivalentTo(new[] { "FULL.nwfs386", "metadata.ini" }), - "NWFS386 minimal-image surface must be exactly the documented opaque pair: FULL.nwfs386 + metadata.ini."); - - var outDir = Path.Combine(Path.GetTempPath(), "Nwfs386Stub_" + Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(outDir); - try { - using var ms2 = new MemoryStream(image, writable: false); - d.Extract(ms2, outDir, password: null, files: null); - var fullPath = Path.Combine(outDir, "FULL.nwfs386"); - Assert.That(File.Exists(fullPath), Is.True, "Extract must produce FULL.nwfs386."); - var roundTrip = File.ReadAllBytes(fullPath); - Assert.That(roundTrip, Is.EqualTo(image), - "FULL.nwfs386 must round-trip the magic-only input byte-for-byte."); - } finally { - Directory.Delete(outDir, recursive: true); - } + [Test, Category("HappyPath")] + public void Descriptor_AdvertisesTheBackedRwSurface() { + var descriptor = new Nwfs386FormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.True); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf(), + "NWFS386 must not claim wipe until all free/salvage/suballocation regions can be proven safely."); + }); + } + + [Test, Category("HappyPath")] + public void Create_ListAndExtract_RoundTripFilesAndDirectories() { + using var image = CreateImage(); + var descriptor = new Nwfs386FormatDescriptor(); + + var entries = descriptor.List(image, null); + + Assert.That(entries.Select(entry => entry.Name), Does.Contain("HELLO.TXT")); + Assert.That(entries.Select(entry => entry.Name), Does.Contain("PUBLIC")); + Assert.That(entries.Select(entry => entry.Name), Does.Contain("PUBLIC/README.DOC")); + + image.Position = 0; + using var opened = descriptor.OpenEntry(image, "PUBLIC/README.DOC", null); + using var copied = new MemoryStream(); + opened.CopyTo(copied); + Assert.That(copied.ToArray(), Is.EqualTo(Readme)); + } + + [Test, Category("HappyPath")] + public void AddReplaceRemove_PreservesGeometryAndLiveBytes() { + using var image = CreateImage(8192); + var descriptor = new Nwfs386FormatDescriptor(); + var replacement = "replacement"u8.ToArray(); + var added = "new file"u8.ToArray(); + + descriptor.Add(image, [ + ArchiveInputInfo.InMemory("HELLO.TXT", replacement), + ArchiveInputInfo.InMemory("NEW.BIN", added), + ]); + descriptor.Remove(image, ["PUBLIC/README.DOC"]); + + var volume = NwfsReader.TryOpen(image.ToArray()); + Assert.That(volume, Is.Not.Null); + Assert.Multiple(() => { + Assert.That(volume!.BlockSize, Is.EqualTo(8192)); + Assert.That(volume.VolumeName, Is.EqualTo("SYS")); + Assert.That(volume.ReadFile("HELLO.TXT"), Is.EqualTo(replacement)); + Assert.That(volume.ReadFile("NEW.BIN"), Is.EqualTo(added)); + Assert.That(volume.ReadFile("PUBLIC/README.DOC"), Is.Null); + }); + } + + [Test, Category("HappyPath")] + public void Purge_LeavesAValidEmptyVolume() { + using var image = CreateImage(); + var descriptor = new Nwfs386FormatDescriptor(); + + ((IArchivePurgeable)descriptor).Purge(image); + + image.Position = 0; + Assert.That(descriptor.List(image, null).Where(entry => !entry.IsDirectory), Is.Empty); + Assert.That(NwfsReader.TryOpen(image.ToArray()), Is.Not.Null); + } + + [Test, Category("HappyPath")] + public void DefragShrinkAndLayout_PreserveContent() { + using var image = CreateImage(4096); + var descriptor = new Nwfs386FormatDescriptor(); + + descriptor.Defragment(image); + var defragged = NwfsReader.TryOpen(image.ToArray())!; + Assert.That(defragged.ReadFile("PUBLIC/README.DOC"), Is.EqualTo(Readme)); + + var padded = image.ToArray().Concat(new byte[128 * 1024]).ToArray(); + using var paddedStream = new MemoryStream(padded, writable: false); + using var shrunk = new MemoryStream(); + descriptor.Shrink(paddedStream, shrunk); + Assert.That(shrunk.Length, Is.LessThan(padded.LongLength)); + Assert.That(NwfsReader.TryOpen(shrunk.ToArray())!.ReadFile("HELLO.TXT"), Is.EqualTo(Hello)); + + image.Position = 0; + using var relaid = new MemoryStream(); + descriptor.RebuildStreaming(image, relaid, new LayoutRebuildOptions { UnitSize = 8192 }); + var layoutVolume = NwfsReader.TryOpen(relaid.ToArray())!; + Assert.Multiple(() => { + Assert.That(layoutVolume.BlockSize, Is.EqualTo(8192)); + Assert.That(layoutVolume.ReadFile("HELLO.TXT"), Is.EqualTo(Hello)); + Assert.That(layoutVolume.ReadFile("PUBLIC/README.DOC"), Is.EqualTo(Readme)); + }); + } + + [Test, Category("HappyPath")] + public void AnalyzeLayout_ReportsCurrentAndCandidateBlockSize() { + using var image = CreateImage(16384); + var descriptor = new Nwfs386FormatDescriptor(); + + var analysis = descriptor.AnalyzeLayout(image); + + Assert.That(analysis.CurrentUnitSize, Is.EqualTo(16384)); + Assert.That(analysis.OptimalUnitSize, Is.AnyOf(1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144)); + Assert.That(analysis.OptimalSlackBytes, Is.LessThanOrEqualTo(analysis.CurrentSlackBytes)); + } + + [Test, Category("ErrorHandling")] + public void Mutation_RejectsCompressionProfileBeforeChangingSource() { + using var image = CreateImage(); + var bytes = image.ToArray(); + var volumeArea = FindVolumeArea(bytes); + var entry = bytes.AsSpan(volumeArea + 32, 60); + var blockValue = BinaryPrimitives.ReadUInt32LittleEndian(entry[44..]); + var blockSize = checked((int)(256u * 1024u / blockValue)); + var firstDirectory = BinaryPrimitives.ReadUInt32LittleEndian(entry[48..]); + var dataArea = volumeArea + 4 * 16384; + var root = checked(dataArea + (int)firstDirectory * blockSize); + bytes[root + 23] |= 0x04; // FILE_COMPRESSION_ON in the ROOT volume flags. + + using var advanced = new MemoryStream(bytes, writable: true); + var before = advanced.ToArray(); + var descriptor = new Nwfs386FormatDescriptor(); + + Assert.Throws(() => descriptor.Add( + advanced, + [ArchiveInputInfo.InMemory("NEW.BIN", "nope"u8.ToArray())])); + Assert.That(advanced.ToArray(), Is.EqualTo(before), "Rejected mutation must leave the source byte-identical."); } - [Test, Category("Stub")] - public void Stub_DoesNotAdvertiseWriteCapability() { - var d = new Nwfs386FormatDescriptor(); - var description = d.Description.ToLowerInvariant(); - Assert.That( - description.Contains("stub") || description.Contains("opaque") - || description.Contains("skeleton") || description.Contains("detection"), - Is.True, - $"NWFS386 Description must honestly flag its stub/detection-only/opaque status. Got: '{d.Description}'."); + private static int FindVolumeArea(byte[] image) { + var magic = "NetWare Volumes\0"u8; + for (var offset = 0; offset + magic.Length <= image.Length; offset += 512) + if (image.AsSpan(offset, magic.Length).SequenceEqual(magic)) + return offset; + throw new InvalidOperationException("Test image contains no NWFS volume table."); } } diff --git a/Compression.Tests/OneFs/OneFsDetectionTests.cs b/Compression.Tests/OneFs/OneFsDetectionTests.cs index 28588d573..05362dfff 100644 --- a/Compression.Tests/OneFs/OneFsDetectionTests.cs +++ b/Compression.Tests/OneFs/OneFsDetectionTests.cs @@ -1,82 +1,177 @@ using System.Text; using Compression.Registry; +using Compression.Registry.Streaming; +using Compression.Tests.Documentation; using FileSystem.OneFs; namespace Compression.Tests.OneFs; /// -/// Stage 0 acceptance gate for : -/// pins the detection magic, surface entry shape, and honest -/// "detection-only" Description. +/// Acceptance tests for the conservative OneFS single-image inspection surface. +/// The tests intentionally do not invent an on-disk filesystem image: Dell's +/// public material documents cluster architecture and physical geometry, not a +/// complete standalone raw-drive serialization that this suite could author. /// [TestFixture] public class OneFsDetectionTests { - private static byte[] BuildMinimal(int payloadLen = 128) { - var image = new byte[16 + payloadLen]; - Encoding.ASCII.GetBytes("OneFS").CopyTo(image.AsSpan(0, 5)); - for (var i = 0; i < payloadLen; i++) image[16 + i] = (byte)(i & 0xFF); + private static byte[] BuildOpaqueImage(int length = 512) { + var image = new byte[length]; + for (var i = 0; i < image.Length; ++i) + image[i] = unchecked((byte)(i * 37 + 11)); return image; } + [Test, Category("Regression")] + public void Descriptor_DoesNotAdvertiseUnverifiedMagic() { + var descriptor = new OneFsFormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor.Id, Is.EqualTo("OneFs")); + Assert.That(descriptor.Extensions, Does.Contain(".onefs")); + Assert.That(descriptor.MagicSignatures, Is.Empty, + "No Dell source was found defining 'OneFS' or 'ONEF' as an offset-zero raw-media signature."); + Assert.That(descriptor.Description, Does.Contain("signature claim was therefore removed").IgnoreCase); + }); + } + [Test, Category("HappyPath")] - public void Detector_IdentifiesByMagic() { - var d = new OneFsFormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("OneFs")); - Assert.That(d.Extensions, Does.Contain(".onefs")); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(2)); - Assert.That(d.MagicSignatures[0].Bytes, Is.EqualTo("OneFS"u8.ToArray())); - Assert.That(d.MagicSignatures[1].Bytes, Is.EqualTo("ONEF"u8.ToArray())); - Assert.That(d, Is.Not.InstanceOf()); + public void List_ReturnsInspectionEntriesWithoutConsumingImage() { + var descriptor = new OneFsFormatDescriptor(); + using var image = new MemoryStream(BuildOpaqueImage(1024)); + image.Position = 137; + + var entries = descriptor.List(image, password: null); + + Assert.Multiple(() => { + Assert.That(entries.Select(entry => entry.Name), Is.EquivalentTo(new[] { + OneFsReader.MetadataEntryName, + OneFsReader.RawImageEntryName, + })); + Assert.That(entries.Single(entry => entry.Name == OneFsReader.RawImageEntryName).OriginalSize, + Is.EqualTo(image.Length)); + Assert.That(image.Position, Is.EqualTo(137), + "Listing an opaque multi-terabyte image must not copy or scan its payload."); + }); } [Test, Category("HappyPath")] - public void List_ReturnsTwoEntries() { - var d = new OneFsFormatDescriptor(); - using var ms = new MemoryStream(BuildMinimal(payloadLen: 256)); - var entries = d.List(ms, password: null); - var names = entries.Select(e => e.Name).ToList(); - Assert.That(names, Is.EquivalentTo(new[] { "metadata.ini", "onefs-volume.bin" })); + public void OpenEntry_StreamsRawImageByteExact() { + var expected = BuildOpaqueImage(4097); + using var image = new MemoryStream(expected, writable: false); + IArchiveFormatOperations operations = new OneFsFormatDescriptor(); + + using var entry = operations.OpenEntry(image, OneFsReader.RawImageEntryName, password: null); + using var copy = new MemoryStream(); + entry.CopyTo(copy); + + Assert.Multiple(() => { + Assert.That(entry, Is.InstanceOf()); + Assert.That(entry.Length, Is.EqualTo(expected.LongLength)); + Assert.That(copy.ToArray(), Is.EqualTo(expected)); + Assert.That(image.CanRead, Is.True, "The descriptor must not take ownership of the caller's archive stream."); + }); } - [Test, Category("Stub")] - public void Description_FlagsDetectionOnly() { - var d = new OneFsFormatDescriptor(); - Assert.That(d.Description.ToLowerInvariant(), Does.Contain("detection-only"), - $"OneFS Description must flag Stage 0 honestly. Got: '{d.Description}'."); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + [Test, Category("HappyPath")] + public void Metadata_ReportsOnlyDocumentedArchitecture() { + using var image = new MemoryStream(BuildOpaqueImage(777)); + using var reader = new OneFsReader(image); + var metadata = reader.Entries.Single(entry => entry.Name == OneFsReader.MetadataEntryName); + var text = Encoding.UTF8.GetString(reader.Extract(metadata)); + + Assert.Multiple(() => { + Assert.That(text, Does.Contain("parse_status=opaque-single-image")); + Assert.That(text, Does.Contain("stage=0")); + Assert.That(text, Does.Contain("authoritative_raw_magic=not_published")); + Assert.That(text, Does.Contain("superblock_magic=known-to-exist-value-not-publicly-verified")); + Assert.That(text, Does.Contain("superblock_locations=multiple-fixed-block-addresses-values-not-publicly-verified")); + Assert.That(text, Does.Contain("superblock_role=references-LIN-master")); + Assert.That(text, Does.Contain($"physical_block_size={OneFsReader.PhysicalBlockSize}")); + Assert.That(text, Does.Contain($"cylinder_group_size={OneFsReader.CylinderGroupSize}")); + Assert.That(text, Does.Contain($"blocks_per_cylinder_group={OneFsReader.BlocksPerCylinderGroup}")); + Assert.That(text, Does.Contain("allocation_tracking=per-cylinder-group-bitmap-serialization-not-published")); + Assert.That(text, Does.Contain("LIN B+ tree")); + Assert.That(text, Does.Contain("protection groups")); + Assert.That(text, Does.Contain("two-phase commit")); + Assert.That(text, Does.Contain("rw_promotion=blocked")); + Assert.That(text, Does.Contain("maintenance=blocked")); + Assert.That(text, Does.Contain("FreeBSD-derived")); + }); } - [Test, Category("Stub")] - public void Description_NamesRoPromotionBlockers() { - // Honest Stage-0 doctrine: Description must surface WHY we cannot promote to R/O. - // Distributed/FEC-striped + no public spec are the two structural blockers; UFS - // ancestry must NOT be advertised as a fallback path. - var d = new OneFsFormatDescriptor(); - var desc = d.Description.ToLowerInvariant(); - Assert.That(desc, Does.Contain("distributed").Or.Contain("clustered"), - "OneFS Description must call out the distributed/clustered architecture."); - Assert.That(desc, Does.Contain("spec").Or.Contain("proprietary"), - "OneFS Description must call out the lack of a public on-disk specification."); - Assert.That(desc, Does.Contain("ufs"), - "OneFS Description must explicitly state UFS-incompatibility so nobody routes images through UfsReader."); + [Test, Category("HappyPath")] + public void LayoutAnalysis_ReportsDocumentedFixedGeometryWithoutReadingImage() { + var descriptor = new OneFsFormatDescriptor(); + using var image = new MemoryStream(BuildOpaqueImage(4096)); + image.Position = 73; + + var layout = (ILayoutOptimizable)descriptor; + var analysis = layout.AnalyzeLayout(image); + + Assert.Multiple(() => { + Assert.That(analysis.ImageSize, Is.EqualTo(image.Length)); + Assert.That(analysis.CurrentUnitSize, Is.EqualTo(OneFsReader.PhysicalBlockSize)); + Assert.That(analysis.OptimalUnitSize, Is.EqualTo(OneFsReader.PhysicalBlockSize)); + Assert.That(analysis.PotentialSavingsBytes, Is.Zero, + "Unknown allocation/slack must not be turned into invented savings."); + Assert.That(analysis.Notes, Has.Some.Contains("33554432")); + Assert.That(analysis.Notes, Has.Some.Contains("4096 blocks")); + Assert.That(image.Position, Is.EqualTo(73), + "Geometry analysis is documentary and must not scan the opaque image."); + Assert.That(FilesystemSupportMatrix.RelaysOut(descriptor), Is.False, + "Analysis-only ILayoutOptimizable must not advertise a working Layout rebuild."); + }); + + using var target = new MemoryStream(); + Assert.That( + () => layout.RebuildStreaming(image, target, new LayoutRebuildOptions()), + Throws.TypeOf(), + "No creator exists, so the default layout rebuild must fail closed."); + } + + [Test, Category("Regression")] + public void LegacyMagicLookingBytes_AreTreatedAsOpaquePayload() { + var imageBytes = BuildOpaqueImage(256); + "OneFSONEF"u8.CopyTo(imageBytes); + using var image = new MemoryStream(imageBytes, writable: false); + using var reader = new OneFsReader(image); + + Assert.Multiple(() => { + Assert.That(reader.Tag, Is.Empty); + Assert.That(reader.TrailingWord, Is.Zero); + Assert.That(reader.ValidHeader, Is.False); + }); + + using var raw = reader.OpenEntry(reader.Entries.Single(entry => entry.Name == OneFsReader.RawImageEntryName)); + var prefix = new byte[9]; + raw.ReadExactly(prefix); + Assert.That(prefix, Is.EqualTo("OneFSONEF"u8.ToArray())); } [Test, Category("Stub")] - public void Metadata_NamesRoPromotionBlockers() { - // Stage-0 metadata.ini must enumerate the R/O-promotion blockers so end-users - // see them without reading source. Pin the structural reasons (distributed, - // FEC, no spec, UFS-incompatible) — not the surface phrasing. - using var ms = new MemoryStream(BuildMinimal(payloadLen: 64)); - var reader = new OneFsReader(ms); - var meta = reader.Entries.First(e => e.Name == "metadata.ini"); - var ini = Encoding.UTF8.GetString(meta.Data).ToLowerInvariant(); - Assert.That(ini, Does.Contain("stage=0"), "metadata.ini must pin stage=0."); - Assert.That(ini, Does.Contain("ro_promotion=blocked"), "metadata.ini must pin ro_promotion=blocked."); - Assert.That(ini, Does.Contain("fec"), "metadata.ini must call out FEC striping as a blocker."); - Assert.That(ini, Does.Contain("lin tree"), "metadata.ini must call out the LIN tree being cluster-wide."); - Assert.That(ini, Does.Contain("spec"), "metadata.ini must call out the missing public spec."); - Assert.That(ini, Does.Contain("ufs"), "metadata.ini must call out UFS-incompatibility at the on-disk layer."); + public void WriteAndDestructiveMaintenanceCapabilities_RemainBlocked() { + var descriptor = new OneFsFormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanTest), Is.False, + "Opaque byte copying cannot validate an undocumented filesystem's integrity."); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf(), + "Documented fixed geometry is safe to analyze even though rewriting it is not."); + Assert.That(descriptor, Is.Not.InstanceOf()); + }); + } + + [Test, Category("Malformed")] + public void Reader_RejectsEmptyImage() { + using var image = new MemoryStream(); + Assert.That(() => new OneFsReader(image), Throws.TypeOf()); } } diff --git a/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs b/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs index 1b40350f6..f128a7769 100644 --- a/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs +++ b/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs @@ -43,6 +43,7 @@ public sealed class FilesystemWriteRoundTripTests { private const string ProbeStem = "PROBE"; private const string Seed = "SEED.BIN"; private const string SeedStem = "SEED"; + private const string TestPassword = "cwb-filesystem-roundtrip"; /// /// Ids whose write claim this fixture cannot exercise, and why. Each entry is a @@ -156,7 +157,8 @@ private void EditByName(string formatId, IArchiveModifiable modifier, byte[] ima var added = Payload(3, 512); byte[] withProbe; try { - withProbe = Mutate(formatId, image, m => modifier.Add(m, [ArchiveInputInfo.InMemory(Probe, added)]), "Add"); + withProbe = Mutate(formatId, image, + m => Add(modifier, formatId, m, [ArchiveInputInfo.InMemory(Probe, added)]), "Add"); } catch (NotSupportedException) { this.EditByOwnEntry(formatId, modifier, image); return; @@ -170,9 +172,9 @@ private void EditByName(string formatId, IArchiveModifiable modifier, byte[] ima // and the bare leaf second. Both together, in one call, is what a strict // remover reads as "one of these is missing". var stored = Names(formatId, withProbe).First(n => Matches(n, ProbeStem)); - var afterRemove = Mutate(formatId, withProbe, m => modifier.Remove(m, [stored]), "Remove"); + var afterRemove = Mutate(formatId, withProbe, m => Remove(modifier, formatId, m, [stored]), "Remove"); if (Names(formatId, afterRemove).Any(n => Matches(n, ProbeStem)) && Leaf(stored) != stored) - afterRemove = Mutate(formatId, withProbe, m => modifier.Remove(m, [Leaf(stored)]), "Remove"); + afterRemove = Mutate(formatId, withProbe, m => Remove(modifier, formatId, m, [Leaf(stored)]), "Remove"); Assert.That(Names(formatId, afterRemove).Any(n => Matches(n, ProbeStem)), Is.False, $"{formatId}: '{stored}' is still listed after Remove."); this.AssertReadsBack(formatId, afterRemove, SeedStem, seed, "after remove (survivor)"); @@ -193,7 +195,7 @@ private void EditByOwnEntry(string formatId, IArchiveModifiable modifier, byte[] var ops = (IArchiveFormatOperations)FormatRegistry.GetArchiveOps(formatId)!; List entries; using (var stream = new MemoryStream(image, writable: false)) - entries = ops.List(stream, null).Where(e => !e.IsDirectory).ToList(); + entries = ops.List(stream, PasswordFor(formatId)).Where(e => !e.IsDirectory).ToList(); // metadata.ini and its kin are rendered from the header, not stored, so // rewriting one asks the format to accept a report of itself as input. @@ -217,7 +219,7 @@ private void EditByOwnEntry(string formatId, IArchiveModifiable modifier, byte[] byte[] edited; try { edited = Mutate(formatId, image, - m => modifier.Add(m, [ArchiveInputInfo.InMemory(candidate.Name, replacement)]), "Add"); + m => Add(modifier, formatId, m, [ArchiveInputInfo.InMemory(candidate.Name, replacement)]), "Add"); } catch (Exception ex) when (ex is NotSupportedException or ArgumentException or InvalidDataException) { refusals.Add($"{candidate.Name}: {ex.Message}"); continue; @@ -291,7 +293,7 @@ public void UnitAddressed_PointAtFixturesThatExist() { private static byte[]? Create(string formatId, IReadOnlyList inputs) { if (FormatRegistry.GetArchiveOps(formatId) is not IArchiveCreatable creator) return null; using var image = new MemoryStream(); - creator.Create(image, inputs, new FormatCreateOptions()); + creator.Create(image, inputs, new FormatCreateOptions { Password = PasswordFor(formatId) }); return image.Length == 0 ? null : image.ToArray(); } @@ -308,9 +310,36 @@ private static byte[] Mutate(string formatId, byte[] image, Action private static List Names(string formatId, byte[] image) { var ops = (IArchiveFormatOperations)FormatRegistry.GetArchiveOps(formatId)!; using var stream = new MemoryStream(image, writable: false); - return ops.List(stream, null).Where(e => !e.IsDirectory).Select(e => e.Name).ToList(); + return ops.List(stream, PasswordFor(formatId)).Where(e => !e.IsDirectory).Select(e => e.Name).ToList(); } + private static void Add( + IArchiveModifiable modifier, + string formatId, + Stream archive, + IReadOnlyList inputs + ) { + var password = PasswordFor(formatId); + if (password is null) + modifier.Add(archive, inputs); + else + modifier.Add(archive, inputs, new ArchiveMutationOptions { Password = password }); + } + + private static void Remove(IArchiveModifiable modifier, string formatId, Stream archive, string[] entryNames) { + var password = PasswordFor(formatId); + if (password is null) + modifier.Remove(archive, entryNames); + else + modifier.Remove(archive, entryNames, new ArchiveMutationOptions { Password = password }); + } + + private static string? PasswordFor(string formatId) + => (FormatRegistry.GetArchiveOps(formatId) as IFormatDescriptor)?.Capabilities + .HasFlag(FormatCapabilities.SupportsPassword) == true + ? TestPassword + : null; + private void AssertReadsBack(string formatId, byte[] image, string stem, byte[] expected, string step) { var extracted = this.ExtractAll(formatId, image); var got = extracted.FirstOrDefault(e => Matches(e.Leaf, stem)).Bytes; @@ -343,7 +372,7 @@ private void AssertEveryEntryExtracts(string formatId, byte[] image, string step try { var ops = (IArchiveFormatOperations)FormatRegistry.GetArchiveOps(formatId)!; using (var stream = new MemoryStream(image, writable: false)) - ops.Extract(stream, outDir, null, null); + ops.Extract(stream, outDir, PasswordFor(formatId), null); return Directory.EnumerateFiles(outDir, "*", SearchOption.AllDirectories) .Select(path => (Path.GetFileName(path), File.ReadAllBytes(path))) .ToList(); diff --git a/Compression.Tests/Refs/RefsBlockCloneExternalTests.cs b/Compression.Tests/Refs/RefsBlockCloneExternalTests.cs new file mode 100644 index 000000000..650132238 --- /dev/null +++ b/Compression.Tests/Refs/RefsBlockCloneExternalTests.cs @@ -0,0 +1,104 @@ +#pragma warning disable CS1591 +using System.ComponentModel; +using System.Diagnostics; +using Category = NUnit.Framework.CategoryAttribute; +using FileSystem.Refs; + +namespace Compression.Tests.Refs; + +/// +/// Optional real-image acceptance gate for the offline ReFS whole-file clone. +/// It intentionally does not download or vendor a ReFS implementation: point +/// CWB_REFS_CLONE_IMAGE at a disposable/reference raw ReFS image and +/// CWB_REFS_FOREFST at an existing forefst.py checkout. The image must contain +/// two distinct, equal-sized, non-empty, cluster-aligned ordinary extent-backed +/// files named by CWB_REFS_CLONE_SOURCE and CWB_REFS_CLONE_DESTINATION. +/// +/// The GPL-3.0-or-later forefst tool is used only as an external behavioral +/// oracle; no source or runtime dependency is incorporated into the product. +/// +[TestFixture] +[Category("ExternalFsInterop")] +public sealed class RefsBlockCloneExternalTests { + private string _tmpDir = null!; + + [SetUp] + public void SetUp() { + this._tmpDir = Path.Combine(Path.GetTempPath(), $"cwb_refs_clone_{Guid.NewGuid():N}"); + Directory.CreateDirectory(this._tmpDir); + } + + [TearDown] + public void TearDown() { + try { Directory.Delete(this._tmpDir, recursive: true); } catch { /* best effort */ } + } + + [Test, CancelAfter(180_000), Category("HappyPath")] + public void RealImage_WholeFileClone_IsReadableByForefst() { + var fixture = Environment.GetEnvironmentVariable("CWB_REFS_CLONE_IMAGE"); + var forefst = Environment.GetEnvironmentVariable("CWB_REFS_FOREFST"); + var source = Environment.GetEnvironmentVariable("CWB_REFS_CLONE_SOURCE") ?? "clone-source.bin"; + var destination = Environment.GetEnvironmentVariable("CWB_REFS_CLONE_DESTINATION") ?? "clone-destination.bin"; + + if (string.IsNullOrWhiteSpace(fixture) || !File.Exists(fixture)) + Assert.Ignore("Set CWB_REFS_CLONE_IMAGE to a real raw ReFS image containing the clone source/destination files."); + if (string.IsNullOrWhiteSpace(forefst) || !File.Exists(forefst)) + Assert.Ignore("Set CWB_REFS_FOREFST to forefst.py for the external ReFS reader oracle."); + + var image = Path.Combine(this._tmpDir, "clone.refs"); + File.Copy(fixture!, image); + var beforeSource = ExtractWithForefst(forefst!, image, source, Path.Combine(this._tmpDir, "source-before.bin")); + + using (var stream = File.Open(image, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) + RefsOfflineBlockCloner.CloneWholeFile(stream, source, destination); + + var afterSource = ExtractWithForefst(forefst!, image, source, Path.Combine(this._tmpDir, "source-after.bin")); + var afterDestination = ExtractWithForefst(forefst!, image, destination, Path.Combine(this._tmpDir, "destination-after.bin")); + + Assert.Multiple(() => { + Assert.That(afterSource, Is.EqualTo(beforeSource), "External reader sees changed source bytes after clone."); + Assert.That(afterDestination, Is.EqualTo(beforeSource), "External reader does not see destination as a byte-identical block clone."); + }); + } + + private static byte[] ExtractWithForefst( + string forefst, + string image, + string path, + string outputPath) { + var python = Environment.GetEnvironmentVariable("CWB_PYTHON") + ?? (OperatingSystem.IsWindows() ? "python" : "python3"); + var start = new ProcessStartInfo { + FileName = python, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + start.ArgumentList.Add(forefst); + start.ArgumentList.Add(image); + start.ArgumentList.Add("extract"); + start.ArgumentList.Add(path); + + Process process; + try { + process = Process.Start(start) + ?? throw new InvalidOperationException("Could not start the forefst process."); + } catch (Win32Exception) { + Assert.Ignore($"Python executable '{python}' is unavailable for the forefst oracle."); + return []; + } + + using (process) { + var stderr = process.StandardError.ReadToEndAsync(); + using (var output = File.Create(outputPath)) + process.StandardOutput.BaseStream.CopyTo(output); + process.WaitForExit(); + var error = stderr.GetAwaiter().GetResult(); + Assert.That(process.ExitCode, Is.Zero, + $"forefst could not extract '{path}' from the ReFS image:\n{error}"); + } + + return File.ReadAllBytes(outputPath); + } +} diff --git a/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs b/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs index 821561f57..928d72dca 100644 --- a/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs +++ b/Compression.Tests/Refs/RefsBlockRefcountCodecTests.cs @@ -80,6 +80,102 @@ public void BuildKey_UsesAlignedRangeAndFixedCount() { }); } + [Test, Category("HappyPath")] + public void BuildFreshValue_InitializesRangeStampAndZeroTotal() { + var row = RefsBlockRefcountCodec.BuildFreshValue(0x123400UL, modificationStamp: 0xE5); + + Assert.Multiple(() => { + Assert.That(row, Has.Length.EqualTo(0x820)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(row.AsSpan(0x00, 8)), Is.EqualTo(0x123400UL)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(row.AsSpan(0x08, 8)), Is.EqualTo(0x400UL)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(row.AsSpan(0x10, 8)), Is.EqualTo(0xE5UL)); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(row.AsSpan(0x18, 4)), Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(row.AsSpan(0x81C, 4)), Is.Zero); + Assert.That(RefsBlockRefcountCodec.HasValidTotal(row), Is.True); + Assert.That(RefsBlockRefcountCodec.IsUnflaggedZeroRow(row), Is.True); + }); + } + + [Test, Category("HappyPath")] + public void AddCloneReferences_FreshRowMaterializesImplicitOwnerAsCountTwo() { + var row = RefsBlockRefcountCodec.BuildFreshValue(0x4000UL, modificationStamp: 0x99); + + var changed = RefsBlockRefcountCodec.AddCloneReferences( + row, + new Dictionary { [7] = 1 }); + + Assert.Multiple(() => { + Assert.That(RefsBlockRefcountCodec.ReadCount(changed, 7), Is.EqualTo(2)); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(changed.AsSpan(0x18, 4)), Is.EqualTo(2)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(changed.AsSpan(0x10, 8)), Is.EqualTo(0x99UL)); + }); + } + + [Test, Category("HappyPath")] + public void AddCloneReferences_ExistingRowZeroSlotStillMaterializesCountTwo() { + var row = BuildRow(0x8000); + WriteRaw(row, 3, 2); // another shared cluster makes the sparse row already exist + RefsBlockRefcountCodec.RefreshTotal(row); + + var changed = RefsBlockRefcountCodec.AddCloneReferences( + row, + new Dictionary { [9] = 1 }); + + Assert.Multiple(() => { + Assert.That(RefsBlockRefcountCodec.ReadCount(changed, 3), Is.EqualTo(2)); + Assert.That(RefsBlockRefcountCodec.ReadCount(changed, 9), Is.EqualTo(2)); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(changed.AsSpan(0x18, 4)), Is.EqualTo(4)); + }); + } + + [Test, Category("HappyPath")] + public void AddCloneReferences_TrackedSlotIncrementsWithoutImplicitBaseline() { + var row = BuildRow(0xC000); + WriteRaw(row, 12, 2); + RefsBlockRefcountCodec.RefreshTotal(row); + + var changed = RefsBlockRefcountCodec.AddCloneReferences( + row, + new Dictionary { [12] = 1 }); + + Assert.That(RefsBlockRefcountCodec.ReadCount(changed, 12), Is.EqualTo(3)); + } + + [Test, Category("HappyPath")] + public void AddCloneReferences_FlaggedZeroSlotDoesNotAssumeOrdinaryOwner() { + var row = BuildRow(0x10000); + WriteRaw(row, 5, RefsBlockRefcountCodec.DedupManagedMask); + RefsBlockRefcountCodec.RefreshTotal(row); + + var changed = RefsBlockRefcountCodec.AddCloneReferences( + row, + new Dictionary { [5] = 1 }); + + Assert.That(RefsBlockRefcountCodec.ReadRaw(changed, 5), Is.EqualTo(0x8001)); + } + + [Test, Category("ErrorHandling")] + public void AddCloneReferences_RejectsNonPositiveReferenceCount() { + var row = BuildRow(0x14000); + + Assert.Multiple(() => { + Assert.Throws(() => RefsBlockRefcountCodec.AddCloneReferences( + row, + new Dictionary { [1] = 0 })); + Assert.Throws(() => RefsBlockRefcountCodec.AddCloneReferences( + row, + new Dictionary { [1] = -1 })); + }); + } + + [Test, Category("ErrorHandling")] + public void FreshRowBuilders_RejectUnalignedRangeStart() { + Assert.Multiple(() => { + Assert.Throws(() => RefsBlockRefcountCodec.BuildKey(0x123401UL)); + Assert.Throws(() => RefsBlockRefcountCodec.BuildFreshValue(0x123401UL, 1)); + }); + } + private static byte[] BuildRow(ulong start) { var row = new byte[RefsBlockRefcountCodec.NormalValueSize]; BinaryPrimitives.WriteUInt64LittleEndian(row.AsSpan(0, 8), start); diff --git a/Compression.Tests/Support/MaintenanceOperationProbe.cs b/Compression.Tests/Support/MaintenanceOperationProbe.cs index 5b59b645e..c36f738c2 100644 --- a/Compression.Tests/Support/MaintenanceOperationProbe.cs +++ b/Compression.Tests/Support/MaintenanceOperationProbe.cs @@ -22,6 +22,7 @@ namespace Compression.Tests.Support; /// internal static class MaintenanceOperationProbe { public const string ProbeName = "PROBE.BIN"; + private const string TestPassword = "cwb-maintenance-probe"; private static readonly byte[] ProbeData = CreateProbeData(); private static readonly string ProbeDigest = Digest(ProbeData); @@ -58,8 +59,17 @@ public static ProbeImage CreateImage(string formatId, string workDirectory) { var image = Path.Combine(workDirectory, "probe.img"); try { - ArchiveOperations.Create(image, [new ArchiveInput(sourcePath, ProbeName)], - new CompressionOptions(), format, null); + // Unencrypted by default. Defragment(Stream) and Purge take no credentials, so a probe + // encrypted merely because the format *supports* passwords is one those verbs cannot read -- + // which is every classic archive format. Only a format that refuses to be created without + // one gets the password, on the retry below. + try { + ArchiveOperations.Create(image, [new ArchiveInput(sourcePath, ProbeName)], + new CompressionOptions(), format, null); + } catch when (PasswordFor(ops!) != null) { + ArchiveOperations.Create(image, [new ArchiveInput(sourcePath, ProbeName)], + new CompressionOptions { Password = PasswordFor(ops!) }, format, null); + } } catch (Exception ex) { Assert.Fail($"{formatId}: advertises a maintenance verb but neither creates the standard probe image " + $"nor declares the input inadmissible through IArchiveWriteConstraints: {ex.GetType().Name}: {ex.Message}"); @@ -93,15 +103,16 @@ public static void AssertProbeFilesAbsent(IArchiveFormatOperations ops, Stream i public static IReadOnlyList ListFiles(IArchiveFormatOperations ops, Stream image) { if (image.CanSeek) image.Position = 0; - return ops.List(image, null).Where(entry => !entry.IsDirectory).ToArray(); + return ops.List(image, PasswordFor(ops)).Where(entry => !entry.IsDirectory).ToArray(); } private static IReadOnlyList ProbeEntries(IArchiveFormatOperations ops, Stream image) { var matches = new List(); + var password = PasswordFor(ops); foreach (var entry in ListFiles(ops, image)) { try { if (image.CanSeek) image.Position = 0; - using var payload = ops.OpenEntry(image, entry.Name, null); + using var payload = ops.OpenEntry(image, entry.Name, password); if (Digest(payload) == ProbeDigest) matches.Add(entry); } catch { @@ -112,6 +123,11 @@ private static IReadOnlyList ProbeEntries(IArchiveFormatOperat return matches; } + private static string? PasswordFor(IArchiveFormatOperations ops) + => (ops as IFormatDescriptor)?.Capabilities.HasFlag(FormatCapabilities.SupportsPassword) == true + ? TestPassword + : null; + private static byte[] CreateProbeData() { var result = new byte[4096]; for (var i = 0; i < result.Length; ++i) diff --git a/Compression.Tests/Tux3/Tux3Tests.cs b/Compression.Tests/Tux3/Tux3Tests.cs index 58c98b852..55cbda2cc 100644 --- a/Compression.Tests/Tux3/Tux3Tests.cs +++ b/Compression.Tests/Tux3/Tux3Tests.cs @@ -7,15 +7,19 @@ namespace Compression.Tests.Tux3; [TestFixture] public class Tux3Tests { - private static byte[] BuildNativeImage(bool legacy2012 = false) { - var image = new byte[16 * 1024]; + private static byte[] BuildNativeImage( + bool legacy2012 = false, + ulong volBlocks = 0x1234, + ushort blockBits = 12, + int imageLength = 16 * 1024) { + var image = new byte[imageLength]; var super = image.AsSpan(Tux3Reader.SuperblockOffset, Tux3Reader.DiskSuperSize); (legacy2012 ? Tux3Reader.Legacy2012Magic : Tux3Reader.Magic).CopyTo(super); BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x08, 8), 0x0123_4567_89AB_CDEFUL); BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x10, 8), 0x1020_3040_5060_7080UL); - BinaryPrimitives.WriteUInt16BigEndian(super.Slice(0x18, 2), 12); - BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x20, 8), 0x0000_0000_0000_1234UL); + BinaryPrimitives.WriteUInt16BigEndian(super.Slice(0x18, 2), blockBits); + BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x20, 8), volBlocks); BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x28, 8), 0x0001_0000_0000_0042UL); BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x30, 8), 0x8002_0000_0000_0043UL); BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x38, 8), 0x0000_0000_0000_0040UL); @@ -75,7 +79,7 @@ public void FormerPrivateTux3SuprMagic_IsRejected() { } [Test, Category("Spec")] - public void Descriptor_AdvertisesOnlyTheNativeMetadataSurface() { + public void Descriptor_AdvertisesNativeMetadataAndSafeMaintenanceSurface() { var descriptor = new Tux3FormatDescriptor(); Assert.Multiple(() => { @@ -85,8 +89,12 @@ public void Descriptor_AdvertisesOnlyTheNativeMetadataSurface() { Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.SupportsMultipleEntries), Is.False); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); + Assert.That(descriptor, Is.InstanceOf()); Assert.That(descriptor, Is.Not.InstanceOf()); Assert.That(descriptor, Is.Not.InstanceOf()); + Assert.That(descriptor, Is.Not.InstanceOf()); Assert.That(descriptor.Description, Does.Contain("native big-endian superblock")); }); } @@ -114,4 +122,121 @@ public void Descriptor_ExtractsNativeMetadataWithoutInventedFileTable() { Directory.Delete(output, recursive: true); } } + + [Test, Category("Spec")] + public void ExtentMap_ReservesUndecodedVolumeAndMarksOnlyExternalTailFree() { + const long declaredLength = 3L << 12; + var image = BuildNativeImage(volBlocks: 3); + using var stream = new MemoryStream(image, writable: false); + + var extents = new Tux3FormatDescriptor().EnumerateExtents(stream).ToArray(); + + Assert.That(extents, Has.Length.EqualTo(2)); + Assert.Multiple(() => { + Assert.That(extents[0].Offset, Is.Zero); + Assert.That(extents[0].Length, Is.EqualTo(declaredLength)); + Assert.That(extents[0].Kind, Is.EqualTo(DefragBlockKind.MetadataReserved)); + Assert.That(extents[1].Offset, Is.EqualTo(declaredLength)); + Assert.That(extents[1].Length, Is.EqualTo(image.LongLength - declaredLength)); + Assert.That(extents[1].Kind, Is.EqualTo(DefragBlockKind.Free)); + }); + } + + [Test, Category("HappyPath")] + public void WipeUnusedSpace_ZeroesOnlyExternalTail() { + const int declaredLength = 3 << 12; + var image = BuildNativeImage(volBlocks: 3); + image[declaredLength - 1] = 0x5A; + Array.Fill(image, (byte)0xA5, declaredLength, image.Length - declaredLength); + using var stream = new MemoryStream(image, writable: true); + var descriptor = new Tux3FormatDescriptor(); + + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(stream, wipeClusterTips: false, wipeDeletedEntries: false); + + Assert.Multiple(() => { + Assert.That(wiped, Is.EqualTo(image.Length - declaredLength)); + Assert.That(image[declaredLength - 1], Is.EqualTo(0x5A)); + Assert.That(image.AsSpan(declaredLength).ToArray(), Is.All.Zero); + }); + } + + [Test, Category("HappyPath")] + public void Shrink_RemovesOnlyExternalTail() { + const int declaredLength = 3 << 12; + var image = BuildNativeImage(volBlocks: 3); + Array.Fill(image, (byte)0xA5, declaredLength, image.Length - declaredLength); + using var input = new MemoryStream(image, writable: false); + using var output = new MemoryStream(); + + new Tux3FormatDescriptor().Shrink(input, output); + + Assert.Multiple(() => { + Assert.That(output.Length, Is.EqualTo(declaredLength)); + Assert.That(output.ToArray(), Is.EqualTo(image[..declaredLength])); + }); + } + + [Test, Category("Regression")] + public void TruncatedDeclaredVolume_IsNeverTreatedAsFreeOrShrunk() { + var image = BuildNativeImage(volBlocks: 5); + image[^1] = 0xA5; + var descriptor = new Tux3FormatDescriptor(); + + using var layoutStream = new MemoryStream(image, writable: false); + var extents = descriptor.EnumerateExtents(layoutStream).ToArray(); + Assert.That(extents, Has.Length.EqualTo(1)); + Assert.Multiple(() => { + Assert.That(extents[0].Offset, Is.Zero); + Assert.That(extents[0].Length, Is.EqualTo(image.LongLength)); + Assert.That(extents[0].Kind, Is.EqualTo(DefragBlockKind.MetadataReserved)); + }); + + var wipeCopy = image.ToArray(); + using var wipeStream = new MemoryStream(wipeCopy, writable: true); + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(wipeStream, wipeClusterTips: false, wipeDeletedEntries: false); + Assert.Multiple(() => { + Assert.That(wiped, Is.Zero); + Assert.That(wipeCopy, Is.EqualTo(image)); + }); + + using var shrinkInput = new MemoryStream(image, writable: false); + using var shrinkOutput = new MemoryStream(); + descriptor.Shrink(shrinkInput, shrinkOutput); + Assert.That(shrinkOutput.ToArray(), Is.EqualTo(image)); + } + + [Test, Category("Regression")] + public void OverflowingDeclaredVolume_FailsClosedInsteadOfTrustingWrappedShift() { + const ushort blockBits = 12; + const ulong volBlocks = (1UL << 52) + 3; + const ulong wrappedLength = volBlocks << blockBits; + var image = BuildNativeImage(volBlocks: volBlocks, blockBits: blockBits); + image[^1] = 0xA5; + var descriptor = new Tux3FormatDescriptor(); + + Assert.That(wrappedLength, Is.EqualTo(3UL << blockBits), + "The regression fixture must wrap a mathematically >64-bit volume into a plausible small boundary."); + + using var layoutStream = new MemoryStream(image, writable: false); + var extents = descriptor.EnumerateExtents(layoutStream).ToArray(); + Assert.That(extents, Has.Length.EqualTo(1)); + Assert.Multiple(() => { + Assert.That(extents[0].Offset, Is.Zero); + Assert.That(extents[0].Length, Is.EqualTo(image.LongLength)); + Assert.That(extents[0].Kind, Is.EqualTo(DefragBlockKind.MetadataReserved)); + }); + + var wipeCopy = image.ToArray(); + using var wipeStream = new MemoryStream(wipeCopy, writable: true); + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(wipeStream, wipeClusterTips: false, wipeDeletedEntries: false); + Assert.Multiple(() => { + Assert.That(wiped, Is.Zero); + Assert.That(wipeCopy, Is.EqualTo(image)); + }); + + using var shrinkInput = new MemoryStream(image, writable: false); + using var shrinkOutput = new MemoryStream(); + descriptor.Shrink(shrinkInput, shrinkOutput); + Assert.That(shrinkOutput.ToArray(), Is.EqualTo(image)); + } } diff --git a/Compression.Tests/UefiFv/UefiFvMaintenanceTests.cs b/Compression.Tests/UefiFv/UefiFvMaintenanceTests.cs new file mode 100644 index 000000000..fdcc524d4 --- /dev/null +++ b/Compression.Tests/UefiFv/UefiFvMaintenanceTests.cs @@ -0,0 +1,129 @@ +using System.Buffers.Binary; +using Compression.Registry; +using FileFormat.UefiFv; + +namespace Compression.Tests.UefiFv; + +[TestFixture] +public sealed class UefiFvMaintenanceTests { + private const string DriverName = "11223344-5566-7788-99aa-bbccddeeff00_DRIVER.bin"; + private const string RawName = "01234567-89ab-cdef-0123-456789abcdef_RAW.bin"; + + [Test, Category("RoundTrip")] + public void Create_LargeFile_UsesFfs3Header2() { + var payload = Enumerable.Range(0, 0x1000000).Select(i => (byte)(i * 17)).ToArray(); + var descriptor = new UefiFvFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, [ArchiveInputInfo.InMemory(DriverName, payload)], new FormatCreateOptions()); + + var bytes = image.ToArray(); + var fv = UefiFvReader.Read(bytes); + Assert.Multiple(() => { + Assert.That(fv.Header.FileSystemGuid, Is.EqualTo(Guid.Parse("5473C07A-3DCB-4DCA-BD6F-1E9689E7349A"))); + Assert.That(fv.Files, Has.Count.EqualTo(1)); + Assert.That(fv.Files[0].Attributes & 0x01, Is.EqualTo(0x01)); + Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(72 + 20, 4)) & 0x00FFFFFF, Is.Zero); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(bytes.AsSpan(72 + 24, 8)), Is.EqualTo((ulong)payload.Length + 32)); + Assert.That(fv.Files[0].Contents, Is.EqualTo(payload)); + }); + } + + [Test, Category("Regression")] + public void Replace_WhenNoRunFits_IsTransactional() { + var descriptor = new UefiFvFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, new byte[1024])], new FormatCreateOptions()); + ((IArchiveModifiable)descriptor).Add(image, [ArchiveInputInfo.InMemory(RawName, new byte[60000])]); + var before = image.ToArray(); + + Assert.That(() => ((IArchiveModifiable)descriptor).Add(image, + [ArchiveInputInfo.InMemory(DriverName, new byte[12000])]), Throws.InstanceOf()); + Assert.That(image.ToArray(), Is.EqualTo(before), "failed replacement must not erase the previous file"); + } + + [Test, Category("RoundTrip")] + public void Defrag_ClosesErasedHole_AndPreservesPayload() { + var descriptor = new UefiFvFormatDescriptor(); + var first = Enumerable.Range(0, 257).Select(i => (byte)i).ToArray(); + var second = Enumerable.Range(0, 721).Select(i => (byte)(i * 7)).ToArray(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, first), ArchiveInputInfo.InMemory(RawName, second)], new FormatCreateOptions()); + ((IArchiveModifiable)descriptor).Remove(image, [DriverName]); + + ((IArchiveDefragmentable)descriptor).Defragment(image); + + image.Position = 0; + var layout = ((IArchiveLayoutMap)descriptor).EnumerateLayout(image).ToList(); + var used = layout.Single(e => e.Kind == DefragBlockKind.Used); + Assert.That(used.Offset, Is.EqualTo(72)); + AssertFiles(image, (RawName, second)); + } + + [Test, Category("RoundTrip")] + public void Wipe_UsesErasePolarity_AndClearsDeletedRecord() { + var descriptor = new UefiFvFormatDescriptor(); + var deleted = Enumerable.Repeat((byte)0xA5, 257).ToArray(); + var live = Enumerable.Repeat((byte)0x5A, 333).ToArray(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, deleted), ArchiveInputInfo.InMemory(RawName, live)], new FormatCreateOptions()); + + var bytes = image.ToArray(); + bytes[72 + 23] = 0xE8; // erase-polarity-one encoding of EFI_FILE_DELETED reached after DATA_VALID + image.Position = 0; + image.Write(bytes); + image.Position = 0; + + var wiped = ((IWipeEmpty)descriptor).WipeUnusedSpace(image, wipeDeletedEntries: true); + Assert.That(wiped, Is.GreaterThan(0)); + var after = image.ToArray(); + var deletedFootprint = (24 + deleted.Length + 7) & ~7; + Assert.That(after.AsSpan(72, deletedFootprint).ToArray(), Is.All.EqualTo((byte)0xFF)); + AssertFiles(image, (RawName, live)); + } + + [Test, Category("RoundTrip")] + public void Purge_LeavesValidEmptyVolume() { + var descriptor = new UefiFvFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, new byte[512]), ArchiveInputInfo.InMemory(RawName, new byte[128])], + new FormatCreateOptions()); + + ((IArchivePurgeable)descriptor).Purge(image); + + var fv = UefiFvReader.Read(image.ToArray()); + Assert.That(fv.Files.Where(f => f.Type != 0xF0), Is.Empty); + image.Position = 0; + Assert.That(descriptor.List(image, null).Select(e => e.Name), Is.EqualTo(new[] { "metadata.ini" })); + } + + [Test, Category("EdgeCase")] + public void Reader_HonorsErasePolarityZeroStateEncoding() { + var descriptor = new UefiFvFormatDescriptor(); + using var image = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(image, + [ArchiveInputInfo.InMemory(DriverName, new byte[] { 1, 2, 3, 4 })], new FormatCreateOptions()); + var bytes = image.ToArray(); + var attributes = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(44)) & ~0x00000800u; + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(44), attributes); + bytes[72 + 23] = 0x07; + bytes.AsSpan((72 + 24 + 4 + 7) & ~7).Clear(); + + var fv = UefiFvReader.Read(bytes); + Assert.Multiple(() => { + Assert.That(fv.Header.ErasePolarity, Is.False); + Assert.That(fv.Files, Has.Count.EqualTo(1)); + Assert.That(fv.Files[0].Contents, Is.EqualTo(new byte[] { 1, 2, 3, 4 })); + }); + } + + private static void AssertFiles(MemoryStream image, params (string Name, byte[] Data)[] expected) { + var actual = UefiFvReader.Read(image.ToArray()).Files.Where(f => f.Type != 0xF0) + .ToDictionary(f => $"{f.Name:D}_{UefiFvReader.ShortTypeTag(f.Type)}.bin", f => f.Contents, StringComparer.OrdinalIgnoreCase); + Assert.That(actual.Keys, Is.EquivalentTo(expected.Select(e => e.Name))); + foreach (var (name, data) in expected) Assert.That(actual[name], Is.EqualTo(data), name); + } +} diff --git a/Compression.Tests/Xfs/XfsExtendedAttributesTests.cs b/Compression.Tests/Xfs/XfsExtendedAttributesTests.cs new file mode 100644 index 000000000..fa7a8efd4 --- /dev/null +++ b/Compression.Tests/Xfs/XfsExtendedAttributesTests.cs @@ -0,0 +1,61 @@ +using FileSystem.Xfs; + +namespace Compression.Tests.Xfs; + +[TestFixture] +public class XfsExtendedAttributesTests { + + private static MemoryStream BuildImage() { + var writer = new XfsWriter(); + writer.AddFile("brick/data.txt", "payload"u8.ToArray()); + var image = new MemoryStream(); + writer.WriteTo(image); + image.Position = 0; + return image; + } + + [Test, Category("RoundTrip")] + public void TrustedGfid_CreateUpdateRemove_RoundTrips() { + using var image = BuildImage(); + byte[] first = [0x10, 0x32, 0x54, 0x76, 0x98, 0xBA, 0xDC, 0xFE, 1, 2, 3, 4, 5, 6, 7, 8]; + byte[] second = [8, 7, 6, 5, 4, 3, 2, 1, 0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10]; + + XfsExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", first); + Assert.That(XfsExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(first)); + + XfsExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", second); + Assert.That(XfsExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(second)); + + Assert.That(XfsExtendedAttributes.Remove(image, "brick/data.txt", "trusted.gfid"), Is.True); + Assert.That(XfsExtendedAttributes.Read(image, "brick/data.txt").ContainsKey("trusted.gfid"), Is.False); + Assert.That(XfsExtendedAttributes.Remove(image, "brick/data.txt", "trusted.gfid"), Is.False); + } + + [Test, Category("RoundTrip")] + public void TrustedGfid_PreservesFilePayload() { + using var image = BuildImage(); + var gfid = Enumerable.Range(0, 16).Select(i => (byte)i).ToArray(); + + XfsExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", gfid); + + image.Position = 0; + using var reader = new XfsReader(image, leaveOpen: true); + var entry = reader.Entries.Single(entry => entry.Name == "brick/data.txt"); + Assert.Multiple(() => { + Assert.That(reader.Extract(entry), Is.EqualTo("payload"u8.ToArray())); + Assert.That(XfsExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(gfid)); + }); + } + + [Test, Category("ErrorHandling")] + public void OversizedShortFormSet_FailsWithoutChangingExistingAttribute() { + using var image = BuildImage(); + var gfid = Enumerable.Range(0, 16).Select(i => (byte)i).ToArray(); + XfsExtendedAttributes.Set(image, "brick/data.txt", "trusted.gfid", gfid); + + Assert.Throws(() => + XfsExtendedAttributes.Set(image, "brick/data.txt", "trusted.too-large", new byte[128])); + + Assert.That(XfsExtendedAttributes.Read(image, "brick/data.txt")["trusted.gfid"], Is.EqualTo(gfid)); + } +} diff --git a/Compression.UI/Views/DefragmentWindow.xaml.cs b/Compression.UI/Views/DefragmentWindow.xaml.cs index 7b58754c4..ddfeb71a5 100644 --- a/Compression.UI/Views/DefragmentWindow.xaml.cs +++ b/Compression.UI/Views/DefragmentWindow.xaml.cs @@ -1687,7 +1687,7 @@ private void OnWipeEmpty(object sender, RoutedEventArgs e) { /// /// Runs the purge operation: erases all live data from the container, /// leaving a valid but empty archive/image. Realised via - /// over every entry. Distinct from + /// over every entry. Distinct from /// wipe-empty, which only zeros dead space (see docs/ARCHIVE-MODEL.md). /// private void OnPurge(object sender, RoutedEventArgs e) { diff --git a/FileFormats/FileFormat.Cso/CsoFormatDescriptor.cs b/FileFormats/FileFormat.Cso/CsoFormatDescriptor.cs index afe02413a..5ebb0e2eb 100644 --- a/FileFormats/FileFormat.Cso/CsoFormatDescriptor.cs +++ b/FileFormats/FileFormat.Cso/CsoFormatDescriptor.cs @@ -8,345 +8,382 @@ namespace FileFormat.Cso; /// -/// PSP CSO / ZSO compressed ISO image. Layout after the 4-byte magic (CISO for CSO, ZISO -/// for LZ4-compressed ZSO): uint32 header_size, uint64 uncompressed_size, uint32 block_size, -/// uint8 version, uint8 align, uint16 reserved, then an index table of N = uncompressed_size / -/// block_size + 1 uint32 entries (high bit = stored/uncompressed, low 31 bits = file offset). -/// -/// This descriptor surfaces each compressed block as a raw blob — it does NOT decompress -/// the blocks (consumers can further process with zlib for CSO or LZ4 for ZSO). -/// -/// References: -/// -/// https://github.com/unknownbrackets/maxcso — maxcso — maintained CSO/ZSO tool; its docs describe the CSO v1/v2 and ZSO layouts -/// The format originates in PSP homebrew (ciso); there is no official Sony documentation -/// +/// PSP CSO v1/v2 and ZSO compressed ISO image. /// +/// +/// The synthetic block entries expose logical, decompressed block contents. Replacing one therefore +/// round-trips through the same bytes a consumer of the ISO sees, while container maintenance can +/// canonicalize physical compression, index alignment and stale tail bytes independently. +/// public sealed class CsoFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, - IArchiveCreatable, IArchiveModifiable { - /// - /// Gets the id. - /// + IArchiveCreatable, IArchiveModifiable, IArchiveDefragmentable, IArchiveShrinkable, + IArchiveLayoutMap, ILayoutOptimizable, IArchivePurgeable, ISyntheticEntryNames { + + private static readonly IReadOnlySet SyntheticNames = new HashSet(StringComparer.OrdinalIgnoreCase) { + "FULL.cso", "FULL.ziso", "metadata.ini", "index.bin", + }; + public string Id => "Cso"; - /// - /// Gets the display name. - /// public string DisplayName => "PSP CSO/ZSO"; - /// - /// Gets the category. - /// public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; - /// - /// Gets the default extension. - /// public string DefaultExtension => ".cso"; - /// - /// Gets the extensions. - /// public IReadOnlyList Extensions => [".cso", ".ziso", ".zso"]; - /// - /// Gets the compound extensions. - /// public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// public IReadOnlyList MagicSignatures => [ new("CISO"u8.ToArray(), Confidence: 0.90), new("ZISO"u8.ToArray(), Confidence: 0.90), ]; - /// - /// Gets the methods. - /// public IReadOnlyList Methods => [ new("stored", "Stored"), - new("deflate", "Deflate"), - new("lz4", "LZ4"), + new("deflate", "Deflate / CSO v1"), + new("lz4", "LZ4 / ZSO"), + new("cso2", "CSO v2 (DEFLATE/LZ4)"), ]; - /// - /// Gets the tar compression format id. - /// public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// - public string Description => "PSP CSO (zlib) / ZSO (LZ4) compressed ISO image."; - - private const uint IndexUncompressedMask = 0x8000_0000u; - private const uint IndexOffsetMask = 0x7FFF_FFFFu; - - private sealed record CsoLayout( - long FullSize, - bool IsZso, - string Magic, - uint HeaderSize, - ulong UncompressedSize, - uint BlockSize, - byte Version, - byte Align, - int BlockCount, - uint[] IndexRaw - ); + public string Description => "PSP CSO v1/v2 (DEFLATE/LZ4) and ZSO (LZ4) compressed ISO image."; + public IReadOnlySet SyntheticEntryNames => SyntheticNames; - /// - /// Lists the entries in the supplied container. - /// public List List(Stream stream, string? password) { - var layout = ReadLayout(stream); + var layout = CsoImage.ReadLayout(stream); var entries = new List(4 + layout.BlockCount); - var ext = layout.IsZso ? "ziso" : "cso"; + var ext = layout.Variant == CsoVariant.Zso ? "ziso" : "cso"; entries.Add(new ArchiveEntryInfo(0, $"FULL.{ext}", layout.FullSize, layout.FullSize, "Stored", false, false, null)); entries.Add(new ArchiveEntryInfo(1, "metadata.ini", 0, 0, "Stored", false, false, null)); entries.Add(new ArchiveEntryInfo(2, "index.bin", layout.IndexRaw.Length * 4L, layout.IndexRaw.Length * 4L, "Stored", false, false, null)); entries.Add(new ArchiveEntryInfo(3, "blocks", 0, 0, "Stored", true, false, null)); - var storedMethod = layout.IsZso ? "LZ4" : "Deflate"; for (var i = 0; i < layout.BlockCount; ++i) { - var (offset, size, isUncompressed) = GetBlockSpan(layout, i); - var method = isUncompressed ? "Stored" : storedMethod; + var (_, physicalLength, encoding) = CsoImage.GetBlockSpan(layout, i); entries.Add(new ArchiveEntryInfo( Index: 4 + i, Name: $"blocks/block_{i:D5}.bin", - OriginalSize: size, - CompressedSize: size, - Method: method, + OriginalSize: CsoImage.LogicalBlockLength(layout, i), + CompressedSize: physicalLength, + Method: MethodName(encoding), IsDirectory: false, IsEncrypted: false, LastModified: null, - Kind: isUncompressed ? "stored" : "compressed")); - // offset intentionally referenced to avoid unused-local warning in contexts that strip. - _ = offset; + Kind: encoding == CsoBlockEncoding.Stored ? "stored" : "compressed")); } return entries; } - /// - /// Decodes the supplied input. - /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var layout = ReadLayout(stream); - var ext = layout.IsZso ? "ziso" : "cso"; + var layout = CsoImage.ReadLayout(stream); + var ext = layout.Variant == CsoVariant.Zso ? "ziso" : "cso"; var fullName = $"FULL.{ext}"; if (Wants(files, fullName)) { + if (layout.FullSize > int.MaxValue) + throw new InvalidDataException("CSO/ZSO whole-image synthetic entry is too large for buffered extraction."); stream.Position = 0; - var buf = new byte[layout.FullSize]; - ReadExact(stream, buf); - WriteFile(outputDir, fullName, buf); + var bytes = new byte[(int)layout.FullSize]; + CsoImage.ReadExact(stream, bytes); + WriteFile(outputDir, fullName, bytes); } if (Wants(files, "metadata.ini")) WriteFile(outputDir, "metadata.ini", Encoding.UTF8.GetBytes(BuildMetadataIni(layout))); if (Wants(files, "index.bin")) { - var idxBytes = new byte[layout.IndexRaw.Length * 4]; + var indexBytes = new byte[layout.IndexRaw.Length * sizeof(uint)]; for (var i = 0; i < layout.IndexRaw.Length; ++i) - BinaryPrimitives.WriteUInt32LittleEndian(idxBytes.AsSpan(i * 4, 4), layout.IndexRaw[i]); - WriteFile(outputDir, "index.bin", idxBytes); + BinaryPrimitives.WriteUInt32LittleEndian(indexBytes.AsSpan(i * sizeof(uint), sizeof(uint)), layout.IndexRaw[i]); + WriteFile(outputDir, "index.bin", indexBytes); } for (var i = 0; i < layout.BlockCount; ++i) { var name = $"blocks/block_{i:D5}.bin"; - if (!Wants(files, name)) continue; - var (offset, size, _) = GetBlockSpan(layout, i); - var data = ReadRange(stream, offset, size); - WriteFile(outputDir, name, data); + if (!Wants(files, name)) + continue; + var decoded = CsoImage.DecodeBlock(stream, layout, i); + var logicalLength = CsoImage.LogicalBlockLength(layout, i); + WriteFile(outputDir, name, decoded.AsSpan(0, logicalLength).ToArray()); } } - private static bool Wants(string[]? files, string name) - => files == null || files.Length == 0 || MatchesFilter(name, files); + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + ArgumentNullException.ThrowIfNull(options); - private static CsoLayout ReadLayout(Stream stream) { - if (!stream.CanSeek) - throw new InvalidDataException("CSO/ZSO descriptor requires a seekable stream."); - stream.Position = 0; - var fullSize = stream.Length; - - Span header = stackalloc byte[24]; - ReadExact(stream, header); - var magic = Encoding.ASCII.GetString(header[..4]); - var isZso = magic switch { - "CISO" => false, - "ZISO" => true, - _ => throw new InvalidDataException("Not a CSO/ZSO image (missing CISO/ZISO magic)."), - }; + var variant = VariantFromCreateOptions(options); + var blockSize = options.GetOptionInt("BlockSize", CsoWriter.DefaultBlockSize); + if (blockSize <= 0) + throw new ArgumentOutOfRangeException(nameof(options), "CSO/ZSO BlockSize must be positive."); - var headerSize = BinaryPrimitives.ReadUInt32LittleEndian(header[4..8]); - var uncompressedSize = BinaryPrimitives.ReadUInt64LittleEndian(header[8..16]); - var blockSize = BinaryPrimitives.ReadUInt32LittleEndian(header[16..20]); - var version = header[20]; - var align = header[21]; - // header[22..24] reserved. - - if (blockSize == 0) - throw new InvalidDataException("CSO/ZSO block_size is zero."); - // Block count is uncompressed_size / block_size + 1 index entries form the table, last entry - // marks end of file. Number of actual blocks is uncompressed_size / block_size (ceil). - var blockCountLong = (long)((uncompressedSize + blockSize - 1) / blockSize); - if (blockCountLong < 0 || blockCountLong > 8_000_000) - throw new InvalidDataException($"CSO/ZSO block count implausible: {blockCountLong}."); - var blockCount = (int)blockCountLong; - - var indexCount = blockCount + 1; - var indexBytes = new byte[indexCount * 4]; - // The index table immediately follows the 24-byte header (header_size is typically 0x18=24). - stream.Position = 24; - ReadExact(stream, indexBytes); - var indexRaw = new uint[indexCount]; - for (var i = 0; i < indexCount; ++i) - indexRaw[i] = BinaryPrimitives.ReadUInt32LittleEndian(indexBytes.AsSpan(i * 4, 4)); - - return new CsoLayout( - FullSize: fullSize, - IsZso: isZso, - Magic: magic, - HeaderSize: headerSize, - UncompressedSize: uncompressedSize, - BlockSize: blockSize, - Version: version, - Align: align, - BlockCount: blockCount, - IndexRaw: indexRaw); + using var payload = CsoInPlaceModifier.CreateScratchStream(); + ulong size = 0; + foreach (var input in inputs) { + if (input.IsDirectory) + continue; + var content = input.ReadContent(); + payload.Write(content); + size = checked(size + (ulong)content.Length); + } + payload.Position = 0; + CsoWriter.Write(output, payload, size, blockSize, variant); } - private static (long Offset, long Size, bool IsUncompressed) GetBlockSpan(CsoLayout layout, int blockIndex) { - var raw = layout.IndexRaw[blockIndex]; - var nextRaw = layout.IndexRaw[blockIndex + 1]; - var isUncompressed = (raw & IndexUncompressedMask) != 0; - var offset = (long)(raw & IndexOffsetMask) << layout.Align; - var nextOffset = (long)(nextRaw & IndexOffsetMask) << layout.Align; - var size = Math.Max(0, nextOffset - offset); - if (offset < 0 || offset > layout.FullSize) - throw new InvalidDataException($"CSO/ZSO block {blockIndex} offset out of range."); - if (offset + size > layout.FullSize) - size = Math.Max(0, layout.FullSize - offset); - return (offset, size, isUncompressed); + /// Replaces logical block_NNNNN.bin entries transactionally. + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + var layout = CsoImage.ReadLayout(archive); + var replacements = new Dictionary(); + foreach (var input in inputs) { + if (input.IsDirectory) + continue; + var index = ParseBlockIndex(input.ArchiveName.Replace('\\', '/')); + if (index < 0) + throw BlockNamespaceOnly(input.ArchiveName, "added"); + if (index >= layout.BlockCount) + throw new ArgumentOutOfRangeException(nameof(inputs), $"CSO/ZSO block index {index} outside [0, {layout.BlockCount})."); + var content = input.ReadContent(); + if (content.Length != layout.BlockSize) + throw new ArgumentException( + $"Replacement block {index} must be exactly block_size ({layout.BlockSize}) bytes; got {content.Length}.", + nameof(inputs)); + replacements[index] = content; + } + if (replacements.Count != 0) + CsoInPlaceModifier.WriteBlocks(archive, replacements); } - private static string BuildMetadataIni(CsoLayout layout) { - var sb = new StringBuilder(); - sb.Append("[Cso]\n"); - sb.Append(CultureInfo.InvariantCulture, $"magic={layout.Magic}\n"); - sb.Append(CultureInfo.InvariantCulture, $"is_zso={(layout.IsZso ? 1 : 0)}\n"); - sb.Append(CultureInfo.InvariantCulture, $"header_size={layout.HeaderSize}\n"); - sb.Append(CultureInfo.InvariantCulture, $"uncompressed_size={layout.UncompressedSize}\n"); - sb.Append(CultureInfo.InvariantCulture, $"block_size={layout.BlockSize}\n"); - sb.Append(CultureInfo.InvariantCulture, $"version={layout.Version}\n"); - sb.Append(CultureInfo.InvariantCulture, $"align={layout.Align}\n"); - sb.Append(CultureInfo.InvariantCulture, $"block_count={layout.BlockCount}\n"); - sb.Append(CultureInfo.InvariantCulture, $"total_size={layout.FullSize}\n"); - return sb.ToString(); + /// Clears logical blocks to zero; it does not edit the ISO 9660 directory tree inside them. + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryNames); + var layout = CsoImage.ReadLayout(archive); + var zero = new byte[checked((int)layout.BlockSize)]; + var replacements = new Dictionary(); + foreach (var name in entryNames) { + var index = ParseBlockIndex(name.Replace('\\', '/')); + if (index < 0) + throw BlockNamespaceOnly(name, "removed"); + if (index >= layout.BlockCount) + throw new ArgumentOutOfRangeException(nameof(entryNames), $"CSO/ZSO block index {index} outside [0, {layout.BlockCount})."); + replacements[index] = zero; + } + if (replacements.Count != 0) + CsoInPlaceModifier.WriteBlocks(archive, replacements); } - private static byte[] ReadRange(Stream stream, long offset, long size) { - if (size <= 0) return []; - if (size > int.MaxValue) - throw new InvalidDataException("CSO/ZSO block too large to extract."); - stream.Position = offset; - var buf = new byte[(int)size]; - ReadExact(stream, buf); - return buf; - } + // ── Maintenance ────────────────────────────────────────────────────── - private static void ReadExact(Stream stream, Span buffer) { - var read = 0; - while (read < buffer.Length) { - var n = stream.Read(buffer[read..]); - if (n <= 0) throw new EndOfStreamException("Unexpected end of CSO/ZSO stream."); - read += n; - } + /// Canonical repack: ordered blocks, align=0, no stale/orphaned body bytes. + public void Defragment(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("CSO/ZSO defragmentation requires a readable, writable, seekable stream.", nameof(archive)); + var layout = CsoImage.ReadLayout(archive); + using var staged = CsoInPlaceModifier.CreateScratchStream(); + Repack(archive, staged, layout, checked((int)layout.BlockSize)); + Commit(staged, archive, truncate: true); } - // ── IArchiveCreatable (CSO v1 only) ─────────────────────────────────── - /// - /// Emits a fresh CSO v1 stream. Inputs are concatenated in supplied order - /// to form the uncompressed payload (the caller is responsible for ensuring - /// the result is a valid PSP ISO if PSP semantics matter). + /// Rebuilds canonically and uses it only when it is smaller; otherwise copies the source through. /// - public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); ArgumentNullException.ThrowIfNull(output); - ArgumentNullException.ThrowIfNull(inputs); - using var payload = new MemoryStream(); - foreach (var input in inputs) { - if (input.IsDirectory) continue; - payload.Write(input.ReadContent()); + using var staged = CsoInPlaceModifier.CreateScratchStream(); + var useStaged = false; + try { + var layout = CsoImage.ReadLayout(input); + Repack(input, staged, layout, checked((int)layout.BlockSize)); + useStaged = staged.Length < input.Length; + } catch { + useStaged = false; } - var bytes = CsoWriter.Build(payload.ToArray()); - output.Write(bytes, 0, bytes.Length); - } - // ── IArchiveModifiable (R/W via block-NNNNN.bin synthetic entries) ──── + output.Position = 0; + output.SetLength(0); + var chosen = useStaged ? staged : input; + chosen.Position = 0; + chosen.CopyTo(output); + } /// - /// Replaces blocks named blocks/block_NNNNN.bin (5-digit zero-padded - /// index) with the supplied payloads. Each input must be exactly the - /// container's block_size bytes. Other input names are ignored. + /// Wipes dead bytes by canonicalizing into the beginning of the same-sized stream and zeroing the + /// now-unindexed tail. If the local codecs cannot reproduce the image no larger than the source, + /// no byte is touched. /// - public void Add(Stream archive, IReadOnlyList inputs) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(inputs); - foreach (var input in inputs) { - if (input.IsDirectory) continue; - var name = input.ArchiveName.Replace('\\', '/'); - var idx = ParseBlockIndex(name); - // A name this cannot place is not a name to pass over: writing nothing and - // raising nothing reports an add that did not happen. - if (idx < 0) - throw new NotSupportedException( - $"CSO: '{input.ArchiveName}' cannot be added. A compressed ISO is edited a block at a " - + "time, so an entry has to be named 'block_NNNNN.bin' for the block it replaces. " - + "Adding a file to the ISO 9660 filesystem inside it is not something this supports."); - CsoInPlaceModifier.WriteBlock(archive, idx, input.ReadContent()); + public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("CSO/ZSO wipe requires a readable, writable, seekable stream.", nameof(image)); + + var originalLength = image.Length; + var layout = CsoImage.ReadLayout(image); + using var staged = CsoInPlaceModifier.CreateScratchStream(); + Repack(image, staged, layout, checked((int)layout.BlockSize)); + if (staged.Length >= originalLength) + return 0; + + var reclaimed = originalLength - staged.Length; + staged.Position = 0; + image.Position = 0; + staged.CopyTo(image); + WriteZeros(image, reclaimed); + image.Flush(); + return reclaimed; + } + + /// Describes the real container byte layout; only bytes after the final index are free. + public IEnumerable EnumerateLayout(Stream archive) { + var layout = CsoImage.ReadLayout(archive); + if (layout.DataStart > 0) + yield return new DefragBlockInfo(0, layout.DataStart, DefragBlockKind.MetadataReserved, "CSO/ZSO header + index"); + + for (var i = 0; i < layout.BlockCount; ++i) { + var (offset, length, encoding) = CsoImage.GetBlockSpan(layout, i); + if (length <= 0) + continue; + yield return new DefragBlockInfo(offset, length, DefragBlockKind.Used, + $"blocks/block_{i:D5}.bin", encoding switch { + CsoBlockEncoding.Stored => DefragBlockClass.Frozen, + CsoBlockEncoding.Lz4 => DefragBlockClass.Cold, + _ => DefragBlockClass.Normal, + }); } + + if (layout.DataEnd < layout.FullSize) + yield return new DefragBlockInfo(layout.DataEnd, layout.FullSize - layout.DataEnd, DefragBlockKind.Free, "unindexed tail"); } - /// - /// "Removes" blocks by writing block_size zero bytes through - /// , which compresses the zero - /// slab to its minimum DEFLATE encoding and zero-pads the on-disk slack. - /// - public void Remove(Stream archive, string[] entryNames) { + public LayoutAnalysis AnalyzeLayout(Stream image) { + var layout = CsoImage.ReadLayout(image); + var trailing = Math.Max(0, layout.FullSize - layout.DataEnd); + var notes = new List { + $"{VariantName(layout.Variant)} with {layout.BlockCount:N0} independently compressed blocks.", + "Canonical rebuilds emit align=0 and remove index-alignment padding/orphaned data without changing logical ISO bytes.", + }; + if (layout.Align != 0) + notes.Add($"Current index_shift={layout.Align}; a rebuild can remove that alignment padding."); + return new LayoutAnalysis { + ImageSize = layout.FullSize, + CurrentUnitSize = checked((int)layout.BlockSize), + CurrentSlackBytes = trailing, + OptimalUnitSize = checked((int)layout.BlockSize), + OptimalSlackBytes = 0, + RequiresRebuild = ["BlockSize", "IndexShift"], + Notes = notes, + }; + } + + /// Reblocks and recompresses while preserving the source CSO/ZSO variant and logical ISO bytes. + public void RebuildStreaming(Stream source, Stream target, LayoutRebuildOptions options) { + ArgumentNullException.ThrowIfNull(source); + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(options); + var layout = CsoImage.ReadLayout(source); + var targetBlockSize = options.UnitSize > 0 ? options.UnitSize : checked((int)layout.BlockSize); + if (targetBlockSize <= 0) + throw new ArgumentOutOfRangeException(nameof(options), "CSO/ZSO target BlockSize must be positive."); + Repack(source, target, layout, targetBlockSize); + } + + /// Leaves a valid empty container of the same CSO/ZSO variant and block geometry. + public void Purge(Stream archive) { ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(entryNames); - var header = CsoInPlaceModifier.ReadHeader(archive); - var zero = new byte[header.BlockSize]; - foreach (var name in entryNames) { - var clean = name.Replace('\\', '/'); - var idx = ParseBlockIndex(clean); - if (idx < 0) - throw new NotSupportedException( - $"CSO: '{name}' cannot be removed. A compressed ISO is edited a block at a time, so an " - + "entry has to be named 'block_NNNNN.bin' for the block it clears. Removing a file from " - + "the ISO 9660 filesystem inside it is not something this supports."); - CsoInPlaceModifier.WriteBlock(archive, idx, zero); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("CSO/ZSO purge requires a readable, writable, seekable stream.", nameof(archive)); + var layout = CsoImage.ReadLayout(archive); + using var emptyInput = new MemoryStream([], writable: false); + using var staged = CsoInPlaceModifier.CreateScratchStream(); + CsoWriter.Write(staged, emptyInput, 0, checked((int)layout.BlockSize), layout.Variant); + Commit(staged, archive, truncate: true); + } + + private static void Repack(Stream source, Stream target, CsoLayout layout, int targetBlockSize) { + source.Position = 0; + using var logical = CsoImage.OpenLogicalStream(source, layout); + target.Position = 0; + target.SetLength(0); + CsoWriter.Write(target, logical, layout.UncompressedSize, targetBlockSize, layout.Variant); + } + + private static void Commit(Stream staged, Stream target, bool truncate) { + staged.Position = 0; + target.Position = 0; + if (truncate) + target.SetLength(0); + staged.CopyTo(target); + target.Flush(); + } + + private static void WriteZeros(Stream stream, long count) { + Span zeros = stackalloc byte[4096]; + while (count > 0) { + var length = (int)Math.Min(count, zeros.Length); + stream.Write(zeros[..length]); + count -= length; } } - /// - /// Parses the trailing block index out of names that look like - /// blocks/block_NNNNN.bin or block_NNNNN.bin. Returns -1 if - /// the name doesn't match the synthetic-entry shape. - /// + private static CsoVariant VariantFromCreateOptions(FormatCreateOptions options) { + var requested = options.GetString("Variant") ?? options.MethodName; + if (string.IsNullOrWhiteSpace(requested)) + return CsoVariant.CsoV1; + return requested.Trim().ToLowerInvariant() switch { + "cso" or "cso1" or "deflate" or "stored" => CsoVariant.CsoV1, + "zso" or "lz4" => CsoVariant.Zso, + "cso2" or "v2" => CsoVariant.CsoV2, + _ => throw new NotSupportedException($"Unsupported CSO/ZSO creation variant '{requested}'."), + }; + } + + private static bool Wants(string[]? files, string name) + => files == null || files.Length == 0 || MatchesFilter(name, files); + + private static string BuildMetadataIni(CsoLayout layout) { + var sb = new StringBuilder(); + sb.Append("[Cso]\n"); + sb.Append(CultureInfo.InvariantCulture, $"magic={layout.Magic}\n"); + sb.Append(CultureInfo.InvariantCulture, $"variant={VariantName(layout.Variant)}\n"); + sb.Append(CultureInfo.InvariantCulture, $"header_size={layout.HeaderSize}\n"); + sb.Append(CultureInfo.InvariantCulture, $"uncompressed_size={layout.UncompressedSize}\n"); + sb.Append(CultureInfo.InvariantCulture, $"block_size={layout.BlockSize}\n"); + sb.Append(CultureInfo.InvariantCulture, $"version={layout.Version}\n"); + sb.Append(CultureInfo.InvariantCulture, $"align={layout.Align}\n"); + sb.Append(CultureInfo.InvariantCulture, $"block_count={layout.BlockCount}\n"); + sb.Append(CultureInfo.InvariantCulture, $"total_size={layout.FullSize}\n"); + return sb.ToString(); + } + + private static string VariantName(CsoVariant variant) => variant switch { + CsoVariant.CsoV1 => "cso1", + CsoVariant.CsoV2 => "cso2", + CsoVariant.Zso => "zso", + _ => "unknown", + }; + + private static string MethodName(CsoBlockEncoding encoding) => encoding switch { + CsoBlockEncoding.Stored => "Stored", + CsoBlockEncoding.Deflate => "Deflate", + CsoBlockEncoding.Lz4 => "LZ4", + _ => "Unknown", + }; + + private static NotSupportedException BlockNamespaceOnly(string name, string verb) + => new( + $"CSO/ZSO: '{name}' cannot be {verb}. A compressed ISO container is edited at logical " + + "block_NNNNN.bin granularity; editing the ISO 9660 filesystem inside it is a separate operation."); + private static int ParseBlockIndex(string name) { var fileName = Path.GetFileNameWithoutExtension(name); const string prefix = "block_"; - if (!fileName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return -1; - var digits = fileName[prefix.Length..]; - return int.TryParse(digits, NumberStyles.Integer, CultureInfo.InvariantCulture, out var idx) && idx >= 0 - ? idx + if (!fileName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + return -1; + return int.TryParse(fileName[prefix.Length..], NumberStyles.Integer, CultureInfo.InvariantCulture, out var index) && index >= 0 + ? index : -1; } } diff --git a/FileFormats/FileFormat.Cso/CsoImage.cs b/FileFormats/FileFormat.Cso/CsoImage.cs new file mode 100644 index 000000000..fe27f7344 --- /dev/null +++ b/FileFormats/FileFormat.Cso/CsoImage.cs @@ -0,0 +1,285 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.IO.Compression; +using System.Text; + +namespace FileFormat.Cso; + +internal enum CsoVariant { + CsoV1, + CsoV2, + Zso, +} + +internal enum CsoBlockEncoding { + Stored, + Deflate, + Lz4, +} + +internal sealed record CsoLayout( + long FullSize, + CsoVariant Variant, + string Magic, + uint HeaderSize, + ulong UncompressedSize, + uint BlockSize, + byte Version, + byte Align, + int BlockCount, + uint[] IndexRaw) { + + public long IndexEnd => CsoWriter.HeaderSize + this.IndexRaw.Length * sizeof(uint); + + public long DataStart => this.IndexRaw.Length == 0 + ? this.IndexEnd + : CsoImage.OffsetOf(this.IndexRaw[0], this.Align); + + public long DataEnd => CsoImage.OffsetOf(this.IndexRaw[^1], this.Align); +} + +/// Shared structural parser and logical block reader for CSO v1/v2 and ZSO. +internal static class CsoImage { + internal const int MaximumBlockCount = 8_000_000; + + public static CsoLayout ReadLayout(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead || !stream.CanSeek) + throw new InvalidDataException("CSO/ZSO parsing requires a readable, seekable stream."); + if (stream.Length < CsoWriter.HeaderSize) + throw new InvalidDataException("CSO/ZSO image is shorter than its 24-byte header."); + + stream.Position = 0; + Span header = stackalloc byte[CsoWriter.HeaderSize]; + ReadExact(stream, header); + + var magic = Encoding.ASCII.GetString(header[..4]); + var headerSize = BinaryPrimitives.ReadUInt32LittleEndian(header[4..8]); + var uncompressedSize = BinaryPrimitives.ReadUInt64LittleEndian(header[8..16]); + var blockSize = BinaryPrimitives.ReadUInt32LittleEndian(header[16..20]); + var version = header[20]; + var align = header[21]; + + var variant = (magic, version) switch { + ("CISO", 0 or 1) => CsoVariant.CsoV1, + ("CISO", 2) => CsoVariant.CsoV2, + ("ZISO", 1) => CsoVariant.Zso, + ("CISO", _) => throw new NotSupportedException($"Unsupported CSO version {version}."), + ("ZISO", _) => throw new NotSupportedException($"Unsupported ZSO version {version}; ZSO defines version 1."), + _ => throw new InvalidDataException("Not a CSO/ZSO image (missing CISO/ZISO magic)."), + }; + + if (variant is CsoVariant.CsoV2 or CsoVariant.Zso && headerSize != CsoWriter.HeaderSize) + throw new InvalidDataException($"{magic} v{version} requires header_size=24, got {headerSize}."); + if (blockSize == 0) + throw new InvalidDataException("CSO/ZSO block_size is zero."); + if (blockSize > int.MaxValue) + throw new InvalidDataException($"CSO/ZSO block_size {blockSize} is too large for managed buffers."); + if (align > 31) + throw new InvalidDataException($"CSO/ZSO index_shift {align} is not representable safely."); + + var blockCount64 = uncompressedSize / blockSize + (uncompressedSize % blockSize == 0 ? 0UL : 1UL); + if (blockCount64 > MaximumBlockCount) + throw new InvalidDataException($"CSO/ZSO block count implausible: {blockCount64}."); + var blockCount = checked((int)blockCount64); + var indexCount = checked(blockCount + 1); + var indexBytesLength = checked(indexCount * sizeof(uint)); + if (CsoWriter.HeaderSize + (long)indexBytesLength > stream.Length) + throw new InvalidDataException("CSO/ZSO index table extends past end of file."); + + var indexBytes = new byte[indexBytesLength]; + stream.Position = CsoWriter.HeaderSize; + ReadExact(stream, indexBytes); + var index = new uint[indexCount]; + for (var i = 0; i < index.Length; ++i) + index[i] = BinaryPrimitives.ReadUInt32LittleEndian(indexBytes.AsSpan(i * sizeof(uint), sizeof(uint))); + + if (variant == CsoVariant.CsoV2 && (index[^1] & CsoWriter.IndexUncompressedFlag) != 0) + throw new InvalidDataException("CSO v2 final index entry must not carry the method flag."); + + var minimumDataOffset = CsoWriter.HeaderSize + (long)indexBytesLength; + long previous = -1; + for (var i = 0; i < index.Length; ++i) { + var offset = OffsetOf(index[i], align); + if (offset < minimumDataOffset) + throw new InvalidDataException($"CSO/ZSO index {i} points inside the header/index table."); + if (offset < previous) + throw new InvalidDataException($"CSO/ZSO index is not monotonic at entry {i}."); + if (offset > stream.Length) + throw new InvalidDataException($"CSO/ZSO index {i} points past end of file."); + previous = offset; + } + + return new CsoLayout( + stream.Length, variant, magic, headerSize, uncompressedSize, blockSize, version, align, + blockCount, index); + } + + public static (long Offset, long Length, CsoBlockEncoding Encoding) GetBlockSpan(CsoLayout layout, int blockIndex) { + ArgumentNullException.ThrowIfNull(layout); + if ((uint)blockIndex >= (uint)layout.BlockCount) + throw new ArgumentOutOfRangeException(nameof(blockIndex)); + + var raw = layout.IndexRaw[blockIndex]; + var offset = OffsetOf(raw, layout.Align); + var next = OffsetOf(layout.IndexRaw[blockIndex + 1], layout.Align); + var length = next - offset; + if (length < 0 || next > layout.FullSize) + throw new InvalidDataException($"CSO/ZSO block {blockIndex} has an invalid physical span."); + + var encoding = layout.Variant switch { + CsoVariant.CsoV1 => (raw & CsoWriter.IndexUncompressedFlag) != 0 + ? CsoBlockEncoding.Stored : CsoBlockEncoding.Deflate, + CsoVariant.Zso => (raw & CsoWriter.IndexUncompressedFlag) != 0 + ? CsoBlockEncoding.Stored : CsoBlockEncoding.Lz4, + CsoVariant.CsoV2 => length >= layout.BlockSize + ? CsoBlockEncoding.Stored + : (raw & CsoWriter.IndexUncompressedFlag) != 0 ? CsoBlockEncoding.Lz4 : CsoBlockEncoding.Deflate, + _ => throw new InvalidDataException("Unknown CSO/ZSO variant."), + }; + return (offset, length, encoding); + } + + public static int LogicalBlockLength(CsoLayout layout, int blockIndex) { + if ((uint)blockIndex >= (uint)layout.BlockCount) + throw new ArgumentOutOfRangeException(nameof(blockIndex)); + var start = checked((ulong)blockIndex * layout.BlockSize); + var remaining = layout.UncompressedSize - start; + return checked((int)Math.Min(remaining, layout.BlockSize)); + } + + /// + /// Decodes one block to its full block_size slab. The final logical block may use only a prefix; + /// writers conventionally pad the remainder before compression and consumers truncate by + /// uncompressed_size. + /// + public static byte[] DecodeBlock(Stream stream, CsoLayout layout, int blockIndex) { + var (offset, length, encoding) = GetBlockSpan(layout, blockIndex); + if (length > int.MaxValue) + throw new InvalidDataException($"CSO/ZSO block {blockIndex} is too large to decode."); + var encoded = new byte[(int)length]; + stream.Position = offset; + ReadExact(stream, encoded); + + var blockSize = checked((int)layout.BlockSize); + switch (encoding) { + case CsoBlockEncoding.Stored: { + var logicalLength = LogicalBlockLength(layout, blockIndex); + if (encoded.Length < logicalLength) + throw new InvalidDataException($"Stored CSO/ZSO block {blockIndex} is shorter than its logical bytes."); + var result = new byte[blockSize]; + encoded.AsSpan(0, Math.Min(encoded.Length, blockSize)).CopyTo(result); + return result; + } + case CsoBlockEncoding.Deflate: + return InflateExact(encoded, blockSize, blockIndex); + case CsoBlockEncoding.Lz4: + return Lz4BlockCodec.Decompress(encoded, blockSize); + default: + throw new InvalidDataException("Unknown CSO/ZSO block encoding."); + } + } + + public static Stream OpenLogicalStream(Stream image, CsoLayout layout, IReadOnlyDictionary? replacements = null) + => new LogicalStream(image, layout, replacements); + + internal static long OffsetOf(uint rawIndex, byte align) { + var units = rawIndex & CsoWriter.IndexOffsetMask; + var offset = (ulong)units << align; + if (offset > long.MaxValue) + throw new InvalidDataException("CSO/ZSO index offset exceeds the supported stream range."); + return (long)offset; + } + + private static byte[] InflateExact(ReadOnlySpan source, int expectedSize, int blockIndex) { + using var input = new MemoryStream(source.ToArray(), writable: false); + using var deflate = new DeflateStream(input, CompressionMode.Decompress, leaveOpen: false); + var output = new byte[expectedSize]; + var written = 0; + while (written < output.Length) { + var read = deflate.Read(output, written, output.Length - written); + if (read <= 0) + break; + written += read; + } + if (written != expectedSize) + throw new InvalidDataException( + $"DEFLATE block {blockIndex} decoded to {written} bytes, expected block_size {expectedSize}."); + return output; + } + + internal static void ReadExact(Stream stream, Span destination) { + var read = 0; + while (read < destination.Length) { + var count = stream.Read(destination[read..]); + if (count <= 0) + throw new EndOfStreamException("Unexpected end of CSO/ZSO stream."); + read += count; + } + } + + private sealed class LogicalStream( + Stream image, + CsoLayout layout, + IReadOnlyDictionary? replacements) : Stream { + private int _blockIndex; + private byte[]? _block; + private int _offsetInBlock; + private long _position; + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => checked((long)layout.UncompressedSize); + public override long Position { + get => this._position; + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + => this.Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) { + if (buffer.IsEmpty || this._position >= this.Length) + return 0; + + var total = 0; + while (!buffer.IsEmpty && this._position < this.Length) { + if (this._block == null) { + if (this._blockIndex >= layout.BlockCount) + throw new InvalidDataException("CSO/ZSO logical stream ended before uncompressed_size."); + if (replacements != null && replacements.TryGetValue(this._blockIndex, out var replacement)) { + if (replacement.Length != layout.BlockSize) + throw new InvalidDataException( + $"Replacement block {this._blockIndex} has {replacement.Length} bytes; expected {layout.BlockSize}."); + this._block = replacement; + } else { + this._block = DecodeBlock(image, layout, this._blockIndex); + } + this._offsetInBlock = 0; + } + + var logicalLength = LogicalBlockLength(layout, this._blockIndex); + var available = logicalLength - this._offsetInBlock; + var take = Math.Min(buffer.Length, available); + this._block.AsSpan(this._offsetInBlock, take).CopyTo(buffer); + buffer = buffer[take..]; + this._offsetInBlock += take; + this._position += take; + total += take; + + if (this._offsetInBlock == logicalLength) { + this._block = null; + this._offsetInBlock = 0; + ++this._blockIndex; + } + } + return total; + } + + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } +} diff --git a/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs b/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs index ad3a6103b..d205fa3d9 100644 --- a/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs +++ b/FileFormats/FileFormat.Cso/CsoInPlaceModifier.cs @@ -1,263 +1,70 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; namespace FileFormat.Cso; /// -/// In-place block-level mutator for PSP CSO v1 images. Lets callers replace -/// individual decompressed blocks without rewriting the whole container. -/// -/// Semantics for : -/// -/// If the new compressed payload fits inside the old block's on-disk -/// slot, the payload is written at the same offset and trailing slack -/// is zero-padded — the index table and every other block's bytes are -/// unchanged. -/// Otherwise the payload is appended at the current end of stream and -/// the block's index entry is updated to point at the new location. -/// The old in-place bytes become orphaned (defrag-recoverable). -/// -/// -/// -/// CSO v2 / ZSO (LZ4) are out of scope; only the v1 header layout with -/// align=0 is supported. The modifier refuses to operate on streams whose -/// header reports a different version or a non-zero align (because the -/// offset-shift semantics would silently misplace the new block). +/// Block-level mutator for CSO v1/v2 and ZSO images. /// +/// +/// A CSO index is ordered by physical block position and each block's physical size is derived from +/// the following index entry. Growing a middle block in place therefore cannot be represented by +/// changing only that entry. Edits are staged as a canonical whole-container repack and committed +/// only after the replacement has been encoded successfully. This also removes stale padding and +/// supports non-zero index shifts without trying to preserve their allocation slack. +/// public static class CsoInPlaceModifier { - /// - /// Replaces block 's content with - /// (which must be exactly - /// block_size bytes long, matching the container's geometry). The - /// payload is DEFLATE-compressed; if the result is smaller than the slab - /// it's written compressed, otherwise stored uncompressed (with the index - /// entry's high bit set). + /// Replaces one logical block. must be exactly block_size + /// bytes; for a partial final block only its logical prefix is retained, matching CSO semantics. /// - /// Seekable read/write stream over the CSO image. - /// 0-based block index. Must be in - /// [0, block_count). - /// Exactly block_size bytes. - public static void WriteBlock(Stream image, int blockIndex, ReadOnlySpan newUncompressedData) { - ArgumentNullException.ThrowIfNull(image); - if (!image.CanSeek) - throw new InvalidOperationException("CSO in-place modify requires a seekable stream."); - if (!image.CanWrite) - throw new InvalidOperationException("CSO in-place modify requires a writable stream."); - - var header = ReadHeader(image); - if (blockIndex < 0 || blockIndex >= header.BlockCount) - throw new ArgumentOutOfRangeException(nameof(blockIndex), - $"CSO block index {blockIndex} outside [0, {header.BlockCount})."); - if (newUncompressedData.Length != header.BlockSize) - throw new ArgumentException( - $"New block payload must be exactly block_size ({header.BlockSize}) bytes; got {newUncompressedData.Length}.", - nameof(newUncompressedData)); - - // Compress (or accept verbatim) the new payload. - var deflated = CsoWriter.Deflate(newUncompressedData); - byte[] payload; - bool storedUncompressed; - if (deflated.Length < header.BlockSize) { - payload = deflated; - storedUncompressed = false; - } else { - payload = newUncompressedData.ToArray(); - storedUncompressed = true; - } - - // Determine old block's on-disk extent. - var oldRaw = header.IndexRaw[blockIndex]; - var oldNextRaw = header.IndexRaw[blockIndex + 1]; - var oldOffset = (long)(oldRaw & CsoWriter.IndexOffsetMask) << header.Align; - var oldEndOffset = (long)(oldNextRaw & CsoWriter.IndexOffsetMask) << header.Align; - var oldSize = oldEndOffset - oldOffset; - - if (payload.Length <= oldSize) { - // In-place: write payload at oldOffset, zero trailing slack. - image.Position = oldOffset; - image.Write(payload, 0, payload.Length); - var slack = (int)(oldSize - payload.Length); - if (slack > 0) - image.Write(new byte[slack], 0, slack); - - // Update flag bit on the index entry (length unchanged → next offset - // is also unchanged, so we only need to rewrite this one entry). - var newRaw = (uint)(oldRaw & CsoWriter.IndexOffsetMask); - if (storedUncompressed) - newRaw |= CsoWriter.IndexUncompressedFlag; - WriteIndexEntry(image, blockIndex, newRaw); - } else { - // Append-at-EOF: orphan the old in-place bytes, write payload at EOF, - // patch this block's index entry to point at the new offset. - var appendOffset = image.Length; - if (appendOffset > CsoWriter.IndexOffsetMask) - throw new InvalidOperationException( - "CSO image would exceed 2 GiB after append — would overflow a 31-bit offset."); - image.Position = appendOffset; - image.Write(payload, 0, payload.Length); + public static void WriteBlock(Stream image, int blockIndex, ReadOnlySpan newUncompressedData) + => WriteBlocks(image, new Dictionary { [blockIndex] = newUncompressedData.ToArray() }); - // The sentinel entry (index[blockCount]) is the on-disk EOF marker; it - // changes by the new payload length minus the existing zero-length sliver - // that lived between the old EOF and the previous sentinel. Recompute - // from the post-write stream length to stay consistent. - var newSentinelRaw = (uint)image.Length; - // The block boundary we're moving is the chosen block's start; its - // length is now (newSentinelRaw - appendOffset). For the WriteBlock - // contract to remain valid for the FOLLOWING entries' offsets we must - // also update every index slot that previously pointed past oldOffset - // but lived inside the file — which is none, because we appended. - - var newRaw = (uint)appendOffset; - if (storedUncompressed) - newRaw |= CsoWriter.IndexUncompressedFlag; - WriteIndexEntry(image, blockIndex, newRaw); - - // Update the sentinel (index[blockCount]) to the new EOF so the reader - // computes our block's compressed length as (sentinel - newOffset). - // BUT — the sentinel only acts as the "next offset" for the LAST block. - // For non-last blocks, the next offset is index[blockIndex+1], which - // didn't move (still points at the OLD orphaned-region neighbour). To - // preserve the (next - this) length contract for our moved block, we - // need the block immediately after ours, in index-order, to "see" us - // at the new offset and yield the right length. - // - // Easiest correct approach: when we append, also update index[blockIndex+1] - // to point at the new EOF so reader's (next - this) = (newEOF - appendOffset) - // = payload.Length. But that breaks the neighbour's own length. - // - // Real CSO writers handle this by simply keeping orphans and laying each - // block's index entry pointing to its physical start; the "length" of - // a block is determined either by (next - this) OR by reading until the - // expected slab boundary. The reader in CsoFormatDescriptor uses - // (next - this), so when we append we need to ensure the next entry is - // the sentinel. - // - // Strategy: if blockIndex is NOT the last block, append a copy of the - // following block's CURRENT compressed bytes after our new payload so - // that index[blockIndex+1] = appendOffset + payload.Length still points - // at a valid (now duplicated) copy of that next block. Repeat through - // the end of the index. - RebuildTailFromBlock(image, header, blockIndex, appendOffset, payload.Length); + /// Replaces several logical blocks in one transactional repack. + internal static void WriteBlocks(Stream image, IReadOnlyDictionary replacements) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(replacements); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new InvalidOperationException("CSO/ZSO block modification requires a readable, writable, seekable stream."); + + var layout = CsoImage.ReadLayout(image); + foreach (var (blockIndex, replacement) in replacements) { + if ((uint)blockIndex >= (uint)layout.BlockCount) + throw new ArgumentOutOfRangeException(nameof(replacements), + $"CSO/ZSO block index {blockIndex} outside [0, {layout.BlockCount})."); + if (replacement.Length != checked((int)layout.BlockSize)) + throw new ArgumentException( + $"New block payload must be exactly block_size ({layout.BlockSize}) bytes; got {replacement.Length} for block {blockIndex}.", + nameof(replacements)); } - } - - /// - /// After an append-grow on , copies every - /// block past it to the new tail so that the (next - this) length contract - /// holds for the entire index. The trailing sentinel is updated to the - /// final EOF. - /// - private static void RebuildTailFromBlock( - Stream image, CsoHeader oldHeader, int movedBlockIndex, - long movedBlockNewOffset, int movedBlockNewSize) { - - var cursor = movedBlockNewOffset + movedBlockNewSize; - var newIndex = new uint[oldHeader.IndexRaw.Length]; - Array.Copy(oldHeader.IndexRaw, newIndex, oldHeader.IndexRaw.Length); - // The moved block's entry was already rewritten by the caller; preserve it. - newIndex[movedBlockIndex] = (uint)movedBlockNewOffset - | (oldHeader.IndexRaw[movedBlockIndex] & CsoWriter.IndexUncompressedFlag); + if (replacements.Count == 0) + return; - for (var i = movedBlockIndex + 1; i < oldHeader.BlockCount; ++i) { - // Copy block i's existing compressed bytes from its OLD offset to the new tail. - var oldOff = (long)(oldHeader.IndexRaw[i] & CsoWriter.IndexOffsetMask) << oldHeader.Align; - var oldEnd = (long)(oldHeader.IndexRaw[i + 1] & CsoWriter.IndexOffsetMask) << oldHeader.Align; - var oldLen = (int)(oldEnd - oldOff); - if (oldLen < 0) oldLen = 0; - var buf = new byte[oldLen]; - if (oldLen > 0) { - image.Position = oldOff; - ReadExact(image, buf); - } - image.Position = cursor; - image.Write(buf, 0, oldLen); + using var logical = CsoImage.OpenLogicalStream(image, layout, replacements); + using var staged = CreateScratchStream(); + CsoWriter.Write(staged, logical, layout.UncompressedSize, checked((int)layout.BlockSize), layout.Variant); - if (cursor > CsoWriter.IndexOffsetMask) - throw new InvalidOperationException( - "CSO image would exceed 2 GiB after append — would overflow a 31-bit offset."); - - newIndex[i] = (uint)cursor | (oldHeader.IndexRaw[i] & CsoWriter.IndexUncompressedFlag); - cursor += oldLen; - } - // Sentinel = final EOF. - if (cursor > CsoWriter.IndexOffsetMask) - throw new InvalidOperationException( - "CSO image would exceed 2 GiB after append — would overflow a 31-bit offset."); - newIndex[oldHeader.BlockCount] = (uint)cursor; - image.SetLength(cursor); - - // Persist the rewritten index. - image.Position = CsoWriter.HeaderSize; - Span idxBuf = stackalloc byte[4]; - for (var i = 0; i < newIndex.Length; ++i) { - BinaryPrimitives.WriteUInt32LittleEndian(idxBuf, newIndex[i]); - image.Write(idxBuf); - } + staged.Position = 0; + image.Position = 0; + image.SetLength(0); + staged.CopyTo(image); + image.Flush(); } - // ── Header parsing ──────────────────────────────────────────────────── + // ── Header parsing retained for callers that only need geometry ─────── internal sealed record CsoHeader( uint HeaderSize, ulong UncompressedSize, uint BlockSize, byte Version, byte Align, - int BlockCount, uint[] IndexRaw); + int BlockCount, uint[] IndexRaw, CsoVariant Variant); internal static CsoHeader ReadHeader(Stream image) { - image.Position = 0; - Span hdr = stackalloc byte[CsoWriter.HeaderSize]; - ReadExact(image, hdr); - if (hdr[0] != 'C' || hdr[1] != 'I' || hdr[2] != 'S' || hdr[3] != 'O') - throw new InvalidDataException("Not a CSO v1 image — missing 'CISO' magic."); - var headerSize = BinaryPrimitives.ReadUInt32LittleEndian(hdr.Slice(4, 4)); - var uncompressedSize = BinaryPrimitives.ReadUInt64LittleEndian(hdr.Slice(8, 8)); - var blockSize = BinaryPrimitives.ReadUInt32LittleEndian(hdr.Slice(16, 4)); - var version = hdr[20]; - var align = hdr[21]; - - if (version != 1) - throw new NotSupportedException( - $"CSO in-place modify supports v1 only (got v{version}). CSO v2 / ZSO are deferred."); - if (align != 0) - throw new NotSupportedException( - $"CSO in-place modify requires align=0 (got align={align}). Non-zero align is deferred " + - "because offset shifts would silently misplace a moved block."); - if (blockSize == 0) - throw new InvalidDataException("CSO block_size is zero."); - - var blockCount = (int)((uncompressedSize + blockSize - 1) / blockSize); - var indexCount = blockCount + 1; - var idxBytes = new byte[indexCount * 4]; - image.Position = CsoWriter.HeaderSize; - ReadExact(image, idxBytes); - var indexRaw = new uint[indexCount]; - for (var i = 0; i < indexCount; ++i) - indexRaw[i] = BinaryPrimitives.ReadUInt32LittleEndian(idxBytes.AsSpan(i * 4, 4)); - - return new CsoHeader(headerSize, uncompressedSize, blockSize, version, align, blockCount, indexRaw); - } - - private static void WriteIndexEntry(Stream image, int blockIndex, uint rawEntry) { - image.Position = CsoWriter.HeaderSize + blockIndex * 4L; - Span buf = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32LittleEndian(buf, rawEntry); - image.Write(buf); - } - - private static void ReadExact(Stream s, Span dst) { - var read = 0; - while (read < dst.Length) { - var n = s.Read(dst[read..]); - if (n <= 0) throw new EndOfStreamException("Unexpected end of CSO stream."); - read += n; - } + var layout = CsoImage.ReadLayout(image); + return new CsoHeader( + layout.HeaderSize, layout.UncompressedSize, layout.BlockSize, layout.Version, layout.Align, + layout.BlockCount, layout.IndexRaw, layout.Variant); } - private static void ReadExact(Stream s, byte[] dst) { - var read = 0; - while (read < dst.Length) { - var n = s.Read(dst, read, dst.Length - read); - if (n <= 0) throw new EndOfStreamException("Unexpected end of CSO stream."); - read += n; - } - } + internal static FileStream CreateScratchStream() + => new(Path.Combine(Path.GetTempPath(), "cwb_cso_" + Guid.NewGuid().ToString("N") + ".tmp"), + FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 64 * 1024, FileOptions.DeleteOnClose); } diff --git a/FileFormats/FileFormat.Cso/CsoWriter.cs b/FileFormats/FileFormat.Cso/CsoWriter.cs index 7d500f43d..8c7731c05 100644 --- a/FileFormats/FileFormat.Cso/CsoWriter.cs +++ b/FileFormats/FileFormat.Cso/CsoWriter.cs @@ -5,139 +5,178 @@ namespace FileFormat.Cso; /// -/// Writes a PSP CSO v1 ("CISO") compressed-ISO container from scratch (WORM). -/// -/// Layout (24-byte header + (N+1)·uint32 index + N compressed blocks): -/// -/// $00..$03: magic "CISO" -/// $04..$07: uint32 LE header_size = 24 -/// $08..$0F: uint64 LE uncompressed_size -/// $10..$13: uint32 LE block_size (this writer uses 2048 = ISO 9660 sector) -/// $14: uint8 version = 1 -/// $15: uint8 align (left-shift applied to index offsets; this writer uses 0) -/// $16..$17: uint16 reserved = 0 -/// $18..(header+4·(N+1)): index table. bit 31 set = stored uncompressed. -/// Each block: raw DEFLATE bytes (no zlib header) of one block_size-aligned slab, -/// OR the slab verbatim when the compressed output is not smaller than the slab. -/// -/// -/// -/// ZSO (LZ4) and CSO v2 are out of scope; see . +/// Writes PSP CSO v1/v2 and ZSO compressed-ISO containers. /// +/// +/// CSO v1 uses raw DEFLATE and bit 31 as the stored/uncompressed flag. ZSO keeps the v1 index +/// semantics but uses raw LZ4 blocks. CSO v2 infers stored blocks from an indexed span greater +/// than or equal to block_size and uses bit 31 to select LZ4 (set) or DEFLATE (clear) for shorter +/// compressed spans. +/// public sealed class CsoWriter { - - /// Default block size — 2048 = one ISO 9660 cooked sector. + /// Default block size — one cooked ISO 9660 sector. public const int DefaultBlockSize = 2048; - /// CSO v1 header length in bytes. + /// CSO/ZSO header length in bytes. internal const int HeaderSize = 24; - /// Bit 31 of an index entry: when set, the block is stored uncompressed. + /// Bit 31 of an index entry: stored for v1/ZSO, LZ4-method for compressed CSO v2. internal const uint IndexUncompressedFlag = 0x8000_0000u; - /// Mask for the offset portion of an index entry. + /// Mask for the shifted file-offset portion of an index entry. internal const uint IndexOffsetMask = 0x7FFF_FFFFu; + /// Builds a canonical CSO v1 stream around . + public static byte[] Build(ReadOnlySpan uncompressedData, int blockSize = DefaultBlockSize) + => Build(uncompressedData, blockSize, CsoVariant.CsoV1); + + internal static byte[] Build(ReadOnlySpan uncompressedData, int blockSize, CsoVariant variant) { + using var input = new MemoryStream(uncompressedData.ToArray(), writable: false); + using var output = new MemoryStream(); + Write(output, input, checked((ulong)uncompressedData.Length), blockSize, variant); + return output.ToArray(); + } + /// - /// Builds a CSO v1 stream that, when fully decompressed, yields - /// . Blocks of - /// bytes are each DEFLATE-compressed; if the compressed output is not smaller - /// than the original slab, the slab is stored verbatim and its index entry - /// gets the bit set. + /// Re-encodes into a canonical align=0 CSO/ZSO stream + /// without buffering the whole ISO in memory. /// - public static byte[] Build(ReadOnlySpan uncompressedData, int blockSize = DefaultBlockSize) { + internal static void Write( + Stream output, Stream uncompressedInput, ulong uncompressedSize, int blockSize, CsoVariant variant) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(uncompressedInput); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("CSO/ZSO output must be writable and seekable.", nameof(output)); + if (!uncompressedInput.CanRead) + throw new ArgumentException("CSO/ZSO input must be readable.", nameof(uncompressedInput)); if (blockSize <= 0) throw new ArgumentOutOfRangeException(nameof(blockSize), "block_size must be positive."); - var uncompressedSize = uncompressedData.Length; - var blockCount = (uncompressedSize + blockSize - 1) / blockSize; - var indexCount = blockCount + 1; // trailing sentinel = EOF offset. + var blockSize64 = checked((ulong)blockSize); + var blockCount64 = uncompressedSize / blockSize64 + (uncompressedSize % blockSize64 == 0 ? 0UL : 1UL); + if (blockCount64 > CsoImage.MaximumBlockCount) + throw new InvalidOperationException($"CSO/ZSO block count is too large: {blockCount64}."); + var blockCount = checked((int)blockCount64); + var index = new uint[checked(blockCount + 1)]; + var dataStart = checked(HeaderSize + index.Length * sizeof(uint)); - var output = new MemoryStream(); - // Reserve space for header + index; we'll patch the index after writing blocks. - output.Write(new byte[HeaderSize + indexCount * 4]); + output.Position = 0; + output.SetLength(dataStart); + output.Position = dataStart; - var index = new uint[indexCount]; + var slab = new byte[blockSize]; + ulong consumed = 0; for (var i = 0; i < blockCount; ++i) { - var rawOffset = i * blockSize; - var rawLen = Math.Min(blockSize, uncompressedSize - rawOffset); - - // If the last block is partial, pad it to blockSize with zero bytes so the - // decoder yields blockSize bytes — its caller truncates by uncompressed_size. - var slab = new byte[blockSize]; - uncompressedData.Slice(rawOffset, rawLen).CopyTo(slab); - - var blockOffset = checked((uint)output.Position); - if (blockOffset > IndexOffsetMask) - throw new InvalidOperationException("CSO output exceeds 2 GiB — would overflow a 31-bit offset."); - index[i] = blockOffset; - - var deflated = Deflate(slab); - if (deflated.Length < blockSize) { - output.Write(deflated, 0, deflated.Length); - } else { - // Compressed payload is not smaller than the slab — store verbatim and - // flag the index entry so the reader knows not to inflate. - output.Write(slab, 0, blockSize); - index[i] |= IndexUncompressedFlag; - } + Array.Clear(slab); + var logicalLength = checked((int)Math.Min(blockSize64, uncompressedSize - consumed)); + ReadExact(uncompressedInput, slab.AsSpan(0, logicalLength)); + consumed += checked((uint)logicalLength); + + var offset = CheckedIndexOffset(output.Position); + var (payload, flag) = EncodeBlock(slab, variant); + index[i] = offset | flag; + output.Write(payload); } - // Sentinel index entry = end-of-stream offset (used by the reader to - // compute the last real block's compressed length via nextOffset-offset). - var eofOffset = checked((uint)output.Position); - if (eofOffset > IndexOffsetMask) - throw new InvalidOperationException("CSO output exceeds 2 GiB — would overflow a 31-bit offset."); - index[blockCount] = eofOffset; + if (consumed != uncompressedSize) + throw new InvalidDataException($"CSO/ZSO encoder consumed {consumed} bytes; expected {uncompressedSize}."); - // Now patch the header + index in place. - var bytes = output.ToArray(); - WriteHeader(bytes, uncompressedSize, blockSize); - for (var i = 0; i < indexCount; ++i) - BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(HeaderSize + i * 4, 4), index[i]); + index[blockCount] = CheckedIndexOffset(output.Position); - return bytes; + Span header = stackalloc byte[HeaderSize]; + WriteHeader(header, uncompressedSize, blockSize, variant); + output.Position = 0; + output.Write(header); + Span entry = stackalloc byte[sizeof(uint)]; + foreach (var value in index) { + BinaryPrimitives.WriteUInt32LittleEndian(entry, value); + output.Write(entry); + } + output.Position = output.Length; } - /// - /// Writes the fixed-layout CSO v1 header into the first 24 bytes of - /// . - /// - internal static void WriteHeader(Span target, long uncompressedSize, int blockSize) { - target[0] = (byte)'C'; - target[1] = (byte)'I'; - target[2] = (byte)'S'; - target[3] = (byte)'O'; - BinaryPrimitives.WriteUInt32LittleEndian(target.Slice(4, 4), HeaderSize); - BinaryPrimitives.WriteUInt64LittleEndian(target.Slice(8, 8), (ulong)uncompressedSize); - BinaryPrimitives.WriteUInt32LittleEndian(target.Slice(16, 4), (uint)blockSize); - target[20] = 1; // version - target[21] = 0; // align - target[22] = 0; // reserved - target[23] = 0; // reserved + internal static void WriteHeader(Span target, ulong uncompressedSize, int blockSize, CsoVariant variant) { + if (target.Length < HeaderSize) + throw new ArgumentException("CSO/ZSO header target is too small.", nameof(target)); + target[..HeaderSize].Clear(); + + var magic = variant == CsoVariant.Zso ? "ZISO"u8 : "CISO"u8; + magic.CopyTo(target); + BinaryPrimitives.WriteUInt32LittleEndian(target[4..8], HeaderSize); + BinaryPrimitives.WriteUInt64LittleEndian(target[8..16], uncompressedSize); + BinaryPrimitives.WriteUInt32LittleEndian(target[16..20], checked((uint)blockSize)); + target[20] = variant == CsoVariant.CsoV2 ? (byte)2 : (byte)1; + target[21] = 0; // canonical writer emits byte-granular indexes (no padding/alignment shift). } - /// Raw-DEFLATE encode (no zlib header). + /// Raw-DEFLATE encode (no zlib wrapper). internal static byte[] Deflate(ReadOnlySpan data) { - using var ms = new MemoryStream(); - using (var ds = new DeflateStream(ms, CompressionLevel.Optimal, leaveOpen: true)) - ds.Write(data); - return ms.ToArray(); + using var output = new MemoryStream(); + using (var deflate = new DeflateStream(output, CompressionLevel.Optimal, leaveOpen: true)) + deflate.Write(data); + return output.ToArray(); } - /// Raw-DEFLATE decode back to its original bytes. + /// Raw-DEFLATE decode helper retained for callers that need a bounded decoded slab. internal static byte[] Inflate(ReadOnlySpan data, int expectedSize) { - using var input = new MemoryStream(data.ToArray()); - using var ds = new DeflateStream(input, CompressionMode.Decompress); + using var input = new MemoryStream(data.ToArray(), writable: false); + using var deflate = new DeflateStream(input, CompressionMode.Decompress); var output = new byte[expectedSize]; var written = 0; - while (written < expectedSize) { - var n = ds.Read(output, written, expectedSize - written); - if (n <= 0) break; - written += n; + while (written < output.Length) { + var read = deflate.Read(output, written, output.Length - written); + if (read <= 0) + break; + written += read; } - if (written < expectedSize) + if (written != output.Length) Array.Resize(ref output, written); return output; } + + private static (byte[] Payload, uint Flag) EncodeBlock(ReadOnlySpan slab, CsoVariant variant) { + switch (variant) { + case CsoVariant.CsoV1: { + var compressed = Deflate(slab); + return compressed.Length < slab.Length + ? (compressed, 0u) + : (slab.ToArray(), IndexUncompressedFlag); + } + case CsoVariant.Zso: { + var compressed = Lz4BlockCodec.Compress(slab); + return compressed.Length < slab.Length + ? (compressed, 0u) + : (slab.ToArray(), IndexUncompressedFlag); + } + case CsoVariant.CsoV2: { + var deflate = Deflate(slab); + var lz4 = Lz4BlockCodec.Compress(slab); + if (lz4.Length < deflate.Length && lz4.Length < slab.Length) + return (lz4, IndexUncompressedFlag); + if (deflate.Length < slab.Length) + return (deflate, 0u); + // CSO v2 does not flag stored blocks: span >= block_size is the stored discriminator. + return (slab.ToArray(), 0u); + } + default: + throw new ArgumentOutOfRangeException(nameof(variant)); + } + } + + private static uint CheckedIndexOffset(long position) { + if (position < 0 || position > IndexOffsetMask) + throw new InvalidOperationException( + "CSO/ZSO output exceeds the 31-bit byte-offset range of an align=0 index."); + return checked((uint)position); + } + + private static void ReadExact(Stream input, Span destination) { + var read = 0; + while (read < destination.Length) { + var count = input.Read(destination[read..]); + if (count <= 0) + throw new EndOfStreamException("Uncompressed CSO/ZSO input ended before uncompressed_size."); + read += count; + } + } } diff --git a/FileFormats/FileFormat.Cso/Lz4BlockCodec.cs b/FileFormats/FileFormat.Cso/Lz4BlockCodec.cs new file mode 100644 index 000000000..d984108e6 --- /dev/null +++ b/FileFormats/FileFormat.Cso/Lz4BlockCodec.cs @@ -0,0 +1,171 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; + +namespace FileFormat.Cso; + +/// +/// Minimal raw-LZ4 block codec used by ZSO and CSO v2. +/// +/// +/// Clean-room implementation from the published LZ4 block format. This is deliberately a block +/// codec, not an LZ4 frame codec: CSO/ZSO supply block boundaries and the decoded size themselves. +/// +internal static class Lz4BlockCodec { + private const int HashBits = 16; + private const int HashSize = 1 << HashBits; + private const int MinimumMatch = 4; + private const int MaximumOffset = ushort.MaxValue; + private const int LastLiterals = 5; + private const int MatchFindLimit = 12; + + /// Compresses one independent LZ4 block. + public static byte[] Compress(ReadOnlySpan source) { + if (source.IsEmpty) + return []; + + var table = new int[HashSize]; + Array.Fill(table, -1); + using var output = new MemoryStream(source.Length); + + var anchor = 0; + var current = 0; + var searchLimit = source.Length - MatchFindLimit; + while (current <= searchLimit) { + var hash = Hash(source, current); + var candidate = table[hash]; + table[hash] = current; + + if (candidate < 0 || current - candidate > MaximumOffset || + !source.Slice(candidate, MinimumMatch).SequenceEqual(source.Slice(current, MinimumMatch))) { + ++current; + continue; + } + + // A compliant LZ4 block keeps its final five source bytes literal and starts the final + // match at least twelve bytes before the end. The search limit above enforces the latter; + // this boundary enforces the former. + var matchEnd = current + MinimumMatch; + var candidateEnd = candidate + MinimumMatch; + var maximumMatchEnd = source.Length - LastLiterals; + while (matchEnd < maximumMatchEnd && source[candidateEnd] == source[matchEnd]) { + ++matchEnd; + ++candidateEnd; + } + + var literalLength = current - anchor; + var matchLength = matchEnd - current; + WriteSequence(output, source.Slice(anchor, literalLength), current - candidate, matchLength); + + var previous = current; + current = matchEnd; + anchor = current; + + // Seed hashes inside the match. This is not required for validity, but makes repeated + // structured sectors compress usefully without introducing a heavyweight search chain. + for (var position = previous + 1; position < current && position <= searchLimit; ++position) + table[Hash(source, position)] = position; + } + + WriteLastLiterals(output, source[anchor..]); + return output.ToArray(); + } + + /// Decompresses one raw LZ4 block to exactly bytes. + public static byte[] Decompress(ReadOnlySpan source, int expectedSize) { + if (expectedSize < 0) + throw new ArgumentOutOfRangeException(nameof(expectedSize)); + if (expectedSize == 0) + return []; + + var output = new byte[expectedSize]; + var inputOffset = 0; + var outputOffset = 0; + + while (outputOffset < expectedSize) { + if (inputOffset >= source.Length) + throw new InvalidDataException("Truncated LZ4 block: missing sequence token."); + + var token = source[inputOffset++]; + var literalLength = ReadLength(source, ref inputOffset, token >> 4); + if (literalLength > source.Length - inputOffset || literalLength > expectedSize - outputOffset) + throw new InvalidDataException("Invalid LZ4 block: literal run exceeds its input or output boundary."); + + source.Slice(inputOffset, literalLength).CopyTo(output.AsSpan(outputOffset)); + inputOffset += literalLength; + outputOffset += literalLength; + + // A final sequence consists solely of literals. CSO/ZSO alignment padding may follow it in + // the indexed span, so once the requested decoded block is complete the remaining bytes are + // intentionally ignored. + if (outputOffset == expectedSize) + return output; + + if (source.Length - inputOffset < 2) + throw new InvalidDataException("Truncated LZ4 block: missing match offset."); + var matchOffset = BinaryPrimitives.ReadUInt16LittleEndian(source[inputOffset..]); + inputOffset += 2; + if (matchOffset == 0 || matchOffset > outputOffset) + throw new InvalidDataException("Invalid LZ4 block: match offset points before the decoded prefix."); + + var matchLength = ReadLength(source, ref inputOffset, token & 0x0F) + MinimumMatch; + if (matchLength > expectedSize - outputOffset) + throw new InvalidDataException("Invalid LZ4 block: match run exceeds the decoded block size."); + + var matchSource = outputOffset - matchOffset; + for (var i = 0; i < matchLength; ++i) + output[outputOffset++] = output[matchSource + i]; + } + + return output; + } + + private static void WriteSequence(Stream output, ReadOnlySpan literals, int offset, int matchLength) { + var matchCode = matchLength - MinimumMatch; + var token = (byte)((Math.Min(literals.Length, 15) << 4) | Math.Min(matchCode, 15)); + output.WriteByte(token); + if (literals.Length >= 15) + WriteExtendedLength(output, literals.Length - 15); + output.Write(literals); + + Span offsetBytes = stackalloc byte[2]; + BinaryPrimitives.WriteUInt16LittleEndian(offsetBytes, checked((ushort)offset)); + output.Write(offsetBytes); + if (matchCode >= 15) + WriteExtendedLength(output, matchCode - 15); + } + + private static void WriteLastLiterals(Stream output, ReadOnlySpan literals) { + output.WriteByte((byte)(Math.Min(literals.Length, 15) << 4)); + if (literals.Length >= 15) + WriteExtendedLength(output, literals.Length - 15); + output.Write(literals); + } + + private static void WriteExtendedLength(Stream output, int remaining) { + while (remaining >= 255) { + output.WriteByte(255); + remaining -= 255; + } + output.WriteByte((byte)remaining); + } + + private static int ReadLength(ReadOnlySpan source, ref int offset, int nibble) { + if (nibble < 15) + return nibble; + + var result = 15; + while (true) { + if (offset >= source.Length) + throw new InvalidDataException("Truncated LZ4 block: missing extended length byte."); + var value = source[offset++]; + result = checked(result + value); + if (value != 255) + return result; + } + } + + private static int Hash(ReadOnlySpan source, int offset) { + var value = BinaryPrimitives.ReadUInt32LittleEndian(source.Slice(offset, sizeof(uint))); + return (int)(value * 2654435761u >> (32 - HashBits)); + } +} diff --git a/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs b/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs index 8665c2de9..a2cc08628 100644 --- a/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs +++ b/FileFormats/FileFormat.FirmwareHex/FirmwareHexCommon.cs @@ -37,10 +37,19 @@ private static byte[] BuildMetadata(FirmwareImage image) { sb.Append(CultureInfo.InvariantCulture, $"base_address = 0x{image.BaseAddress:X8}\n"); sb.Append(CultureInfo.InvariantCulture, $"start_address = {(image.StartAddress.HasValue ? $"0x{image.StartAddress.Value:X8}" : "(unspecified)")}\n"); + if (image.StartSegmentAddress is { } segmentedStart) { + var (cs, ip) = segmentedStart; + sb.AppendLine("start_address_kind = segment"); + sb.Append(CultureInfo.InvariantCulture, $"start_segment_cs = 0x{cs:X4}\n"); + sb.Append(CultureInfo.InvariantCulture, $"start_segment_ip = 0x{ip:X4}\n"); + } else if (image.StartAddress.HasValue) { + sb.AppendLine("start_address_kind = linear"); + } for (var i = 0; i < image.Segments.Count; i++) { var (a, d) = image.Segments[i]; + var end = (ulong)a + (uint)d.Length; sb.Append(CultureInfo.InvariantCulture, - $"segment_{i} = 0x{a:X8} .. 0x{a + (uint)d.Length:X8} ({d.Length} bytes)\n"); + $"segment_{i} = 0x{a:X8} .. 0x{end:X8} ({d.Length} bytes)\n"); } return Encoding.UTF8.GetBytes(sb.ToString()); } diff --git a/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs b/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs index 7105155b2..6c7d4b452 100644 --- a/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs +++ b/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs @@ -14,11 +14,13 @@ namespace FileFormat.FirmwareHex; /// and both are fully specified, so writing them is a transcription rather than a /// reconstruction. The reader normalises an image to a single firmware.bin /// plus a rendered metadata.ini; the writer reads that pair back — the -/// payload is the bytes, the metadata supplies the base and start addresses the -/// flat binary cannot carry. +/// payload is the bytes, the metadata supplies the address map and start address +/// information the flat binary cannot carry. /// public static class FirmwareHexWriter { + private readonly record struct SegmentLayout(uint Address, int Length); + /// The name the reader gives the flat payload. public const string PayloadName = "firmware.bin"; @@ -37,12 +39,16 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st byte[]? payload = null; var baseAddress = 0u; uint? startAddress = null; + ushort? startSegmentCs = null; + ushort? startSegmentIp = null; + var segmentLayout = new List(); foreach (var input in inputs) { if (input.IsDirectory) continue; var leaf = Path.GetFileName(input.ArchiveName); if (leaf.Equals(MetadataName, StringComparison.OrdinalIgnoreCase)) { - ReadMetadata(input.ReadContent(), ref baseAddress, ref startAddress); + ReadMetadata(input.ReadContent(), ref baseAddress, ref startAddress, + segmentLayout, ref startSegmentCs, ref startSegmentIp); continue; } // Any other single input is the payload, whatever it is called: a caller @@ -50,33 +56,137 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st payload ??= input.ReadContent(); } - var segments = payload is { Length: > 0 } - ? new List<(uint, byte[])> { (baseAddress, payload) } - : []; - return new FirmwareImage(segments, startAddress, RecordCount: 0, GapCount: 0, - TotalDataBytes: payload?.Length ?? 0, SourceFormat: sourceFormat); + List<(uint Address, byte[] Data)> segments; + if (payload is not { Length: > 0 }) + segments = []; + else if (!TryRestoreSparseSegments(payload, baseAddress, segmentLayout, out segments)) + segments = [(baseAddress, payload)]; + + var image = new FirmwareImage(segments, startAddress, RecordCount: 0, + GapCount: Math.Max(0, segments.Count - 1), + TotalDataBytes: segments.Sum(segment => segment.Data.Length), + SourceFormat: sourceFormat); + + if (startAddress is { } linear && startSegmentCs is { } cs && startSegmentIp is { } ip + && ((uint)cs << 4) + ip == linear) + image = image with { StartSegmentAddress = (cs, ip) }; + + return image; } - /// Reads the two addresses the flat payload cannot carry out of a rendered summary. - private static void ReadMetadata(byte[] metadata, ref uint baseAddress, ref uint? startAddress) { + /// + /// Reads the addresses and segment map that a flat payload cannot carry out of + /// a rendered summary. Unknown metadata is deliberately ignored so older and + /// hand-written summaries remain accepted. + /// + private static void ReadMetadata(byte[] metadata, ref uint baseAddress, ref uint? startAddress, + List segmentLayout, ref ushort? startSegmentCs, ref ushort? startSegmentIp) { foreach (var raw in Encoding.UTF8.GetString(metadata).Split('\n')) { var line = raw.Trim(); var equals = line.IndexOf('=', StringComparison.Ordinal); if (equals < 0) continue; var key = line[..equals].Trim(); var value = line[(equals + 1)..].Trim(); - if (!value.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) continue; - if (!uint.TryParse(value.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var parsed)) continue; - if (key.Equals("base_address", StringComparison.OrdinalIgnoreCase)) baseAddress = parsed; - else if (key.Equals("start_address", StringComparison.OrdinalIgnoreCase)) startAddress = parsed; + + if (key.Equals("base_address", StringComparison.OrdinalIgnoreCase)) { + if (TryParseHexUInt32(value, out var parsed)) baseAddress = parsed; + continue; + } + if (key.Equals("start_address", StringComparison.OrdinalIgnoreCase)) { + if (TryParseHexUInt32(value, out var parsed)) startAddress = parsed; + continue; + } + if (key.Equals("start_segment_cs", StringComparison.OrdinalIgnoreCase)) { + if (TryParseHexUInt16(value, out var parsed)) startSegmentCs = parsed; + continue; + } + if (key.Equals("start_segment_ip", StringComparison.OrdinalIgnoreCase)) { + if (TryParseHexUInt16(value, out var parsed)) startSegmentIp = parsed; + continue; + } + if (key.StartsWith("segment_", StringComparison.OrdinalIgnoreCase) + && TryParseSegmentLayout(value, out var segment)) + segmentLayout.Add(segment); + } + } + + private static bool TryParseHexUInt32(string text, out uint value) { + value = 0; + return text.StartsWith("0x", StringComparison.OrdinalIgnoreCase) + && uint.TryParse(text.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out value); + } + + private static bool TryParseHexUInt16(string text, out ushort value) { + value = 0; + return text.StartsWith("0x", StringComparison.OrdinalIgnoreCase) + && ushort.TryParse(text.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out value); + } + + private static bool TryParseHexUInt64(string text, out ulong value) { + value = 0; + return text.StartsWith("0x", StringComparison.OrdinalIgnoreCase) + && ulong.TryParse(text.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out value); + } + + private static bool TryParseSegmentLayout(string value, out SegmentLayout segment) { + segment = default; + const string separator = " .. "; + var separatorIndex = value.IndexOf(separator, StringComparison.Ordinal); + if (separatorIndex <= 0) return false; + + var startText = value[..separatorIndex].Trim(); + var remainder = value[(separatorIndex + separator.Length)..].Trim(); + var endTokenLength = remainder.IndexOf(' '); + if (endTokenLength < 0) endTokenLength = remainder.Length; + var endText = remainder[..endTokenLength]; + if (!TryParseHexUInt32(startText, out var start) || !TryParseHexUInt64(endText, out var end) + || end < start || end > 0x1_0000_0000UL) + return false; + + var length = end - start; + if (length > int.MaxValue) return false; + segment = new SegmentLayout(start, (int)length); + return true; + } + + /// + /// Restores the sparse runs described by metadata from the reader's flat binary. + /// The layout is used only when it spans the payload exactly; if callers replace + /// firmware.bin with a differently sized binary, stale segment metadata is + /// ignored and the replacement becomes one contiguous run at . + /// + private static bool TryRestoreSparseSegments(byte[] payload, uint baseAddress, + IReadOnlyList layout, out List<(uint Address, byte[] Data)> segments) { + segments = []; + if (layout.Count == 0) return false; + + var ordered = layout.OrderBy(segment => segment.Address).ToArray(); + if (ordered[0].Address != baseAddress) return false; + + ulong previousEnd = baseAddress; + ulong highestEnd = baseAddress; + foreach (var segment in ordered) { + if (segment.Address < previousEnd) return false; + var end = (ulong)segment.Address + (uint)segment.Length; + if (end > 0x1_0000_0000UL) return false; + previousEnd = end; + highestEnd = Math.Max(highestEnd, end); } + if (highestEnd - baseAddress != (ulong)payload.Length) return false; + + foreach (var segment in ordered) { + var offset = (ulong)segment.Address - baseAddress; + if (offset + (uint)segment.Length > (ulong)payload.Length) return false; + segments.Add((segment.Address, payload.AsSpan(checked((int)offset), segment.Length).ToArray())); + } + return true; } /// /// Writes as Intel HEX: type-04 extended-linear-address /// records whenever the high half of the address changes, type-00 data records of /// at most bytes that never straddle a 64 KiB - /// boundary, an optional type-05 start-linear-address record, and the type-01 + /// boundary, an optional type-03 or type-05 start-address record, and the type-01 /// end-of-file record every reader requires. /// public static void WriteIntelHex(Stream output, FirmwareImage image, int bytesPerRecord = 16) { @@ -88,6 +198,10 @@ public static void WriteIntelHex(Stream output, FirmwareImage image, int bytesPe var text = new StringBuilder(); var upper = -1L; foreach (var (segmentAddress, data) in image.Segments) { + if ((ulong)segmentAddress + (uint)data.Length > 0x1_0000_0000UL) + throw new InvalidDataException( + $"IntelHex: segment at 0x{segmentAddress:X8} with {data.Length} bytes exceeds the 32-bit address space."); + var offset = 0; while (offset < data.Length) { var address = segmentAddress + (uint)offset; @@ -106,8 +220,12 @@ public static void WriteIntelHex(Stream output, FirmwareImage image, int bytesPe } } - if (image.StartAddress is { } start) + if (image.StartSegmentAddress is { } segmentedStart) { + var (cs, ip) = segmentedStart; + Record(text, 0, 0x03, [(byte)(cs >> 8), (byte)cs, (byte)(ip >> 8), (byte)ip]); + } else if (image.StartAddress is { } start) { Record(text, 0, 0x05, [(byte)(start >> 24), (byte)(start >> 16), (byte)(start >> 8), (byte)start]); + } Record(text, 0, 0x01, []); var bytes = Encoding.ASCII.GetBytes(text.ToString()); diff --git a/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs b/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs index bc2710aa7..4717c712d 100644 --- a/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs +++ b/FileFormats/FileFormat.FirmwareHex/FirmwareImage.cs @@ -17,6 +17,14 @@ public sealed record FirmwareImage( string SourceFormat ) { + /// + /// Gets the original segmented x86 start address when an Intel HEX type-03 + /// record supplied one. Keeping CS:IP separately matters because many CS:IP + /// pairs map to the same linear address and therefore cannot be reconstructed + /// from alone. + /// + public (ushort CodeSegment, ushort InstructionPointer)? StartSegmentAddress { get; init; } + /// /// Flattens all segments into a single contiguous binary spanning from the /// lowest address to the end of the highest segment. Gaps are filled with @@ -25,11 +33,16 @@ string SourceFormat public byte[] ToFlatBinary(byte fill = 0xFF) { if (this.Segments.Count == 0) return []; var lo = this.Segments.Min(s => s.Address); - var hi = this.Segments.Max(s => s.Address + (uint)s.Data.Length); - var buf = new byte[hi - lo]; + var hi = this.Segments.Max(s => (ulong)s.Address + (uint)s.Data.Length); + var length = hi - lo; + if (length > int.MaxValue) + throw new InvalidDataException( + $"Firmware image spans {length} bytes and cannot be represented as one managed flat byte array."); + + var buf = new byte[(int)length]; if (fill != 0) Array.Fill(buf, fill); foreach (var (addr, data) in this.Segments) - Array.Copy(data, 0, buf, (int)(addr - lo), data.Length); + Array.Copy(data, 0, buf, checked((int)((ulong)addr - lo)), data.Length); return buf; } diff --git a/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs b/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs index 460f3a92a..8f5f929bf 100644 --- a/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs +++ b/FileFormats/FileFormat.FirmwareHex/IntelHexFormatDescriptor.cs @@ -9,15 +9,17 @@ namespace FileFormat.FirmwareHex; /// /// Pseudo-archive descriptor for Intel HEX firmware files. Decodes the ASCII /// records into a flat binary (firmware.bin) and surfaces a -/// metadata.ini with record count, declared start address, and gap count. +/// metadata.ini with record count, declared start address, and sparse +/// segment layout. /// /// References: /// /// Intel "Hexadecimal Object File Format Specification", Rev. A (1988) — the defining document -/// https://en.wikipedia.org/wiki/Intel_HEX — record types and checksum rules +/// https://developerhelp.microchip.com/xwiki/bin/view/software-tools/ipe/sqtp-file-format-specification/intel-hex/ — record layout and checksum rules /// /// -public sealed class IntelHexFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable { +public sealed class IntelHexFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, + IArchiveModifiable, IArchiveDefragmentable { /// /// Gets the id. @@ -36,7 +38,7 @@ public sealed class IntelHexFormatDescriptor : IFormatDescriptor, IArchiveFormat /// public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | - FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; + FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; /// /// Gets the default extension. /// @@ -95,14 +97,45 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? /// /// Writes a fresh Intel HEX file: the single payload input becomes the data /// records, and a metadata.ini alongside it -- the one this descriptor's - /// own reader renders -- supplies the base and start addresses that a flat - /// binary cannot carry. + /// own reader renders -- supplies the sparse segment map and start-address form + /// that a flat binary cannot carry. /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { ArgumentNullException.ThrowIfNull(output); FirmwareHexWriter.WriteIntelHex(output, FirmwareHexWriter.ImageFrom(inputs, "IntelHex")); } + /// + /// Removes all programmed bytes and start-address state while leaving the + /// canonical valid empty Intel HEX document (the EOF record). This overrides + /// the generic pseudo-archive purge because firmware.bin is a rendered + /// view that also exists as a zero-length view of an empty image. + /// + public void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("Intel HEX purge requires a readable, writable, seekable stream.", nameof(archive)); + + archive.Position = 0; + _ = BuildEntries(archive); // validate before committing a destructive edit + + using var rebuilt = new MemoryStream(); + FirmwareHexWriter.WriteIntelHex(rebuilt, + new FirmwareImage([], StartAddress: null, RecordCount: 0, GapCount: 0, TotalDataBytes: 0, SourceFormat: "IntelHex")); + archive.Position = 0; + archive.SetLength(0); + rebuilt.Position = 0; + rebuilt.CopyTo(archive); + archive.Flush(); + archive.Position = 0; + } + + // IArchiveModifiable and IArchiveDefragmentable intentionally use their shared + // verified staged-rebuild implementations. Rebuilding is the native edit model + // for a line-oriented HEX file: there are no allocation structures to patch in + // place, and the rendered metadata now preserves sparse address runs and the + // distinction between type-03 CS:IP and type-05 linear start records. + private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) { using var reader = new StreamReader(stream, Encoding.ASCII, detectEncodingFromByteOrderMarks: true, leaveOpen: true); var text = reader.ReadToEnd(); diff --git a/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs b/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs index b90602dcc..3ad0ecf11 100644 --- a/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs +++ b/FileFormats/FileFormat.FirmwareHex/IntelHexReader.cs @@ -17,6 +17,7 @@ public static FirmwareImage Read(string text) { var segments = new SortedDictionary(); uint extendedBase = 0; uint? startAddr = null; + (ushort CodeSegment, ushort InstructionPointer)? startSegmentAddress = null; var recordCount = 0; var sawEof = false; @@ -24,6 +25,8 @@ public static FirmwareImage Read(string text) { for (var line = reader.ReadLine(); line != null; line = reader.ReadLine()) { line = line.Trim(); if (line.Length == 0 || line[0] != ':') continue; + if (sawEof) + throw new InvalidDataException("IntelHex: record found after the end-of-file record."); if (line.Length < 11) throw new InvalidDataException($"IntelHex: short record '{line}'."); recordCount++; @@ -45,27 +48,38 @@ public static FirmwareImage Read(string text) { $"IntelHex: checksum mismatch (computed 0x{expected:X2}, stored 0x{bytes[^1]:X2}) on '{line}'."); var payload = bytes.AsSpan(4, len).ToArray(); + ValidateRecordShape(type, addr, payload.Length, line); switch (type) { - case 0x00: // data - AddSegment(segments, extendedBase + addr, payload); + case 0x00: { // data + var absoluteAddress = extendedBase + addr; + if ((ulong)absoluteAddress + (uint)payload.Length > 0x1_0000_0000UL) + throw new InvalidDataException($"IntelHex: data record crosses the 32-bit address-space limit in '{line}'."); + AddSegment(segments, absoluteAddress, payload); break; + } case 0x01: // EOF sawEof = true; break; case 0x02: // extended segment address (16-bit segment * 16) extendedBase = (uint)(((payload[0] << 8) | payload[1]) << 4); break; - case 0x03: // start segment address (CS:IP) - startAddr = (uint)(((payload[0] << 24) | (payload[1] << 16)) + ((payload[2] << 8) | payload[3])); + case 0x03: { // start segment address (CS:IP) + var cs = (ushort)((payload[0] << 8) | payload[1]); + var ip = (ushort)((payload[2] << 8) | payload[3]); + startAddr = ((uint)cs << 4) + ip; + startSegmentAddress = (cs, ip); break; + } case 0x04: // extended linear address (high 16 bits) extendedBase = (uint)((payload[0] << 24) | (payload[1] << 16)); break; case 0x05: // start linear address (EIP) startAddr = (uint)((payload[0] << 24) | (payload[1] << 16) | (payload[2] << 8) | payload[3]); + startSegmentAddress = null; break; default: - // Unknown record types are ignored (some vendors extend the format). + // Unknown record types are ignored after checksum validation. Some + // vendor tools extend Intel HEX beyond the six standard record types. break; } } @@ -74,7 +88,25 @@ public static FirmwareImage Read(string text) { throw new InvalidDataException("IntelHex: missing ':00000001FF' end-of-file record."); var (merged, gapCount, totalBytes) = MergeSegments(segments); - return new FirmwareImage(merged, startAddr, recordCount, gapCount, totalBytes, "IntelHex"); + return new FirmwareImage(merged, startAddr, recordCount, gapCount, totalBytes, "IntelHex") { + StartSegmentAddress = startSegmentAddress, + }; + } + + private static void ValidateRecordShape(byte type, ushort address, int payloadLength, string line) { + var expectedPayloadLength = type switch { + 0x01 => 0, + 0x02 or 0x04 => 2, + 0x03 or 0x05 => 4, + _ => -1, + }; + if (expectedPayloadLength >= 0 && payloadLength != expectedPayloadLength) + throw new InvalidDataException( + $"IntelHex: record type 0x{type:X2} requires {expectedPayloadLength} data bytes, got {payloadLength} in '{line}'."); + + if ((type is 0x01 or 0x02 or 0x03 or 0x04 or 0x05) && address != 0) + throw new InvalidDataException( + $"IntelHex: record type 0x{type:X2} requires address 0000, got {address:X4} in '{line}'."); } private static byte[] ParseHexBytes(ReadOnlySpan hex) { @@ -90,10 +122,11 @@ private static byte[] ParseHexBytes(ReadOnlySpan hex) { } private static void AddSegment(SortedDictionary segments, uint addr, byte[] data) { - // Merge with an immediately-preceding segment so consecutive 16-byte records fold into one. + // Merge with an immediately-preceding segment so consecutive data records + // fold into one logical byte run. foreach (var k in segments.Keys) { var existing = segments[k]; - if (k + (uint)existing.Length == addr) { + if ((ulong)k + (uint)existing.Length == addr) { var merged = new byte[existing.Length + data.Length]; Buffer.BlockCopy(existing, 0, merged, 0, existing.Length); Buffer.BlockCopy(data, 0, merged, existing.Length, data.Length); @@ -107,13 +140,13 @@ private static void AddSegment(SortedDictionary segments, uint add internal static (List<(uint Address, byte[] Data)> Segments, int GapCount, int TotalBytes) MergeSegments(SortedDictionary segments) { var list = new List<(uint, byte[])>(); - uint? prevEnd = null; + ulong? prevEnd = null; var gaps = 0; var total = 0; foreach (var (addr, data) in segments) { if (prevEnd.HasValue && addr > prevEnd.Value) gaps++; list.Add((addr, data)); - prevEnd = addr + (uint)data.Length; + prevEnd = (ulong)addr + (uint)data.Length; total += data.Length; } return (list, gaps, total); diff --git a/FileFormats/FileFormat.FirmwareHex/TiTxtFormatDescriptor.cs b/FileFormats/FileFormat.FirmwareHex/TiTxtFormatDescriptor.cs index 79c244c7c..0b83df390 100644 --- a/FileFormats/FileFormat.FirmwareHex/TiTxtFormatDescriptor.cs +++ b/FileFormats/FileFormat.FirmwareHex/TiTxtFormatDescriptor.cs @@ -1,4 +1,5 @@ #pragma warning disable CS1591 +using System.Globalization; using System.Text; using Compression.Registry; using static Compression.Registry.FormatHelpers; @@ -7,84 +8,58 @@ namespace FileFormat.FirmwareHex; /// /// Pseudo-archive descriptor for the TI-TXT firmware text format used by MSP430. -/// Address lines (@HHHH) introduce contiguous byte runs; a single q -/// terminates the file. Extension is intentionally empty — .txt is far -/// too ambiguous — so detection relies on the first non-whitespace byte being -/// @. +/// Address lines (@HHHH) introduce sparse byte runs; a line containing +/// q terminates the file. The archive view exposes the logical firmware as +/// firmware.bin plus a rendered metadata.ini summary. /// /// References: /// -/// Texas Instruments MSP430 programming/bootloader guides — define the TI-TXT format (@addr / data / q) -/// https://srecord.sourceforge.net — SRecord tool suite — documents and converts TI-TXT (srec_ti_txt) +/// https://downloads.ti.com/docs/esd/SPRU513P/Content/SPRU513P_HTML/hex-conversion-utility-description.html — TI Hex Conversion Utility, TI-TXT format +/// https://www.ti.com/lit/pdf/slau131 — Texas Instruments MSP430 programming documentation +/// https://srecord.sourceforge.net/man/man5/srec_ti_txt.5.html — SRecord TI-TXT documentation; used only as an interoperability reference /// /// -public sealed class TiTxtFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable { +public sealed class TiTxtFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, + IArchiveModifiable, IArchiveDefragmentable, IArchiveWriteConstraints, ISyntheticEntryNames { + + private static readonly IReadOnlySet SyntheticNames = + new HashSet(StringComparer.OrdinalIgnoreCase) { FirmwareHexWriter.MetadataName }; - /// - /// Gets the id. - /// public string Id => "TiTxt"; - /// - /// Gets the display name. - /// public string DisplayName => "TI-TXT (MSP430)"; - /// - /// Gets the category. - /// public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | - FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; - /// - /// Gets the default extension. - /// + FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; public string DefaultExtension => ".txt"; - /// - /// Gets the extensions. - /// public IReadOnlyList Extensions => []; - /// - /// Gets the compound extensions. - /// public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// - public IReadOnlyList MagicSignatures => [ - // TI-TXT begins with '@' on the first non-whitespace line; low confidence - // because '@' shows up in lots of text formats (email/yaml/etc). - new([(byte)'@'], Confidence: 0.15), - ]; - /// - /// Gets the methods. - /// + public IReadOnlyList MagicSignatures => [new([(byte)'@'], Confidence: 0.15)]; public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// public AlgorithmFamily Family => AlgorithmFamily.Encoding; - /// - /// Gets the description. - /// public string Description => - "TI-TXT MSP430 firmware text (address/data/q-terminator)."; + "TI-TXT MSP430 sparse firmware text; supports create, replace/remove, purge and canonical rebuild."; + + public IReadOnlySet SyntheticEntryNames => SyntheticNames; + + public bool CanAccept(ArchiveInputInfo input, out string? reason) { + ArgumentNullException.ThrowIfNull(input); + if (!input.IsDirectory) { + reason = null; + return true; + } + + reason = "TI-TXT accepts one firmware payload plus optional metadata.ini; directories are not representable."; + return false; + } + + public long? MaxTotalArchiveSize => null; + public string AcceptedInputsDescription => "one firmware payload plus optional metadata.ini"; - /// - /// Lists the entries in the supplied container. - /// public List List(Stream stream, string? password) => FirmwareHexCommon.BuildArchiveEntries(BuildEntries(stream)); - /// - /// Decodes the supplied input. - /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { foreach (var e in BuildEntries(stream)) { if (files != null && files.Length > 0 && !MatchesFilter(e.Name, files)) continue; @@ -92,20 +67,157 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? } } - /// - /// Writes a fresh TI-TXT file: the single payload input becomes the data lines - /// under an @address taken from a metadata.ini alongside it, and - /// the file ends with the q the format requires. - /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + ValidateInputs(inputs, out _, out _); FirmwareHexWriter.WriteTiTxt(output, FirmwareHexWriter.ImageFrom(inputs, "TiTxt")); } - private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) { + /// + /// Replaces the single logical firmware payload and/or moves its base address. + /// A metadata-only base-address edit shifts every sparse section by the same + /// delta so address holes remain holes rather than becoming explicit 0xFF data. + /// + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + ValidateInputs(inputs, out var payload, out var metadata); + if (payload is null && metadata is null) return; + + var current = ReadImage(archive); + var baseAddress = metadata is null ? current.BaseAddress : ReadBaseAddress(metadata.ReadContent()) ?? current.BaseAddress; + FirmwareImage replacement; + + if (payload is not null) { + var bytes = payload.ReadContent(); + replacement = ImageFromSegments( + bytes.Length == 0 ? [] : [(baseAddress, bytes)]); + } else if (current.Segments.Count == 0 || baseAddress == current.BaseAddress) { + replacement = current; + } else { + replacement = ShiftImage(current, baseAddress); + } + + Rewrite(archive, replacement); + } + + /// Removing the logical firmware payload leaves the valid empty TI-TXT form, q. + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryNames); + if (!entryNames.Any(name => Path.GetFileName(name).Equals(FirmwareHexWriter.PayloadName, StringComparison.OrdinalIgnoreCase))) + return; + Rewrite(archive, ImageFromSegments([])); + } + + /// Erases all live firmware bytes while leaving a valid empty TI-TXT document. + public void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + _ = ReadImage(archive); // reject malformed input before replacing it + Rewrite(archive, ImageFromSegments([])); + } + + /// + /// Canonicalises section ordering and record wrapping without flattening sparse + /// holes. The reader merges only exactly adjacent sections; the writer then + /// emits one address record per contiguous run and 16 bytes per full data line. + /// + public void Defragment(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + Rewrite(archive, ReadImage(archive)); + } + + private static void ValidateInputs(IReadOnlyList inputs, + out ArchiveInputInfo? payload, out ArchiveInputInfo? metadata) { + payload = null; + metadata = null; + foreach (var input in inputs) { + if (input.IsDirectory) + throw new InvalidDataException("TI-TXT cannot store directories."); + if (Path.GetFileName(input.ArchiveName).Equals(FirmwareHexWriter.MetadataName, StringComparison.OrdinalIgnoreCase)) { + if (metadata is not null) + throw new InvalidDataException("TI-TXT accepts at most one metadata.ini input."); + metadata = input; + continue; + } + if (payload is not null) + throw new InvalidDataException("TI-TXT represents one logical firmware payload; multiple payload inputs are ambiguous."); + payload = input; + } + } + + private static uint? ReadBaseAddress(byte[] metadata) { + foreach (var raw in Encoding.UTF8.GetString(metadata).Split('\n')) { + var line = raw.Trim(); + var equals = line.IndexOf('=', StringComparison.Ordinal); + if (equals < 0 || !line[..equals].Trim().Equals("base_address", StringComparison.OrdinalIgnoreCase)) continue; + var value = line[(equals + 1)..].Trim(); + if (value.StartsWith("0x", StringComparison.OrdinalIgnoreCase) && + uint.TryParse(value.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var parsed)) + return parsed; + } + return null; + } + + private static FirmwareImage ShiftImage(FirmwareImage image, uint newBaseAddress) { + var delta = (long)newBaseAddress - image.BaseAddress; + var shifted = new List<(uint Address, byte[] Data)>(image.Segments.Count); + foreach (var (address, data) in image.Segments) { + var target = (long)address + delta; + if (target < 0 || target > uint.MaxValue || (ulong)target + (ulong)data.LongLength > (ulong)uint.MaxValue + 1) + throw new InvalidDataException("TI-TXT base-address edit would move a section outside the 32-bit address space."); + shifted.Add(((uint)target, data)); + } + return ImageFromSegments(shifted); + } + + private static FirmwareImage ImageFromSegments(List<(uint Address, byte[] Data)> segments) { + var totalBytes = 0; + for (var i = 0; i < segments.Count; ++i) checked { totalBytes += segments[i].Data.Length; } + var gaps = 0; + for (var i = 1; i < segments.Count; ++i) + if ((ulong)segments[i - 1].Address + (ulong)segments[i - 1].Data.Length < segments[i].Address) gaps++; + return new FirmwareImage(segments, StartAddress: null, RecordCount: 0, GapCount: gaps, + TotalDataBytes: totalBytes, SourceFormat: "TiTxt"); + } + + private static FirmwareImage ReadImage(Stream stream) { + if (!stream.CanRead) throw new ArgumentException("TI-TXT stream must be readable.", nameof(stream)); + if (stream.CanSeek) stream.Position = 0; using var reader = new StreamReader(stream, Encoding.ASCII, detectEncodingFromByteOrderMarks: true, leaveOpen: true); - var text = reader.ReadToEnd(); - var image = TiTxtReader.Read(text); - return FirmwareHexCommon.BuildEntries(image); + return TiTxtReader.Read(reader.ReadToEnd()); + } + + private static void Rewrite(Stream archive, FirmwareImage image) { + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("TI-TXT mutation requires a readable, writable, seekable stream.", nameof(archive)); + + using var staged = new MemoryStream(); + FirmwareHexWriter.WriteTiTxt(staged, image); + staged.Position = 0; + var reparsed = ReadImage(staged); + if (!SameSparseImage(image, reparsed)) + throw new InvalidDataException("TI-TXT rewrite verification failed: sparse address/data semantics changed."); + + archive.Position = 0; + archive.SetLength(0); + staged.Position = 0; + staged.CopyTo(archive); + archive.SetLength(archive.Position); + archive.Position = 0; } + + private static bool SameSparseImage(FirmwareImage expected, FirmwareImage actual) { + if (expected.Segments.Count != actual.Segments.Count) return false; + for (var i = 0; i < expected.Segments.Count; ++i) { + var left = expected.Segments[i]; + var right = actual.Segments[i]; + if (left.Address != right.Address || !left.Data.AsSpan().SequenceEqual(right.Data)) return false; + } + return true; + } + + private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) => + FirmwareHexCommon.BuildEntries(ReadImage(stream)); } diff --git a/FileFormats/FileFormat.FirmwareHex/TiTxtReader.cs b/FileFormats/FileFormat.FirmwareHex/TiTxtReader.cs index f7b676ce3..fe4e3b1a1 100644 --- a/FileFormats/FileFormat.FirmwareHex/TiTxtReader.cs +++ b/FileFormats/FileFormat.FirmwareHex/TiTxtReader.cs @@ -4,53 +4,93 @@ namespace FileFormat.FirmwareHex; /// -/// Reader for TI-TXT MSP430 text firmware files. Addresses are introduced by -/// @HHHH lines; data lines follow as space-separated hex bytes (typically -/// 16 per line); a single q token terminates the file. +/// Parser for Texas Instruments TI-TXT firmware images. Address records introduce +/// sparse byte runs and a line containing only q terminates the document. /// public sealed class TiTxtReader { - /// Parses a TI-TXT document into a . + /// Parses a TI-TXT document into its sparse firmware image. public static FirmwareImage Read(string text) { - var segments = new SortedDictionary>(); - uint? currentAddr = null; + ArgumentNullException.ThrowIfNull(text); + + var sections = new List<(uint Address, List Data)>(); + List? currentData = null; var sawTerminator = false; var dataLines = 0; using var reader = new StringReader(text); - for (var line = reader.ReadLine(); line != null; line = reader.ReadLine()) { - line = line.Trim(); + for (var raw = reader.ReadLine(); raw != null; raw = reader.ReadLine()) { + var line = raw.Trim(); if (line.Length == 0) continue; - if (line.StartsWith('q') || line.StartsWith('Q')) { + if (sawTerminator) + throw new InvalidDataException($"TiTxt: content after the 'q' termination line ('{line}')."); + + if (line.Equals("q", StringComparison.OrdinalIgnoreCase)) { sawTerminator = true; - break; + currentData = null; + continue; } + if (line.StartsWith('@')) { - if (!uint.TryParse(line.AsSpan(1).Trim(), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var addr)) + var address = line.AsSpan(1).Trim(); + if (address.IsEmpty || !uint.TryParse(address, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var parsed)) throw new InvalidDataException($"TiTxt: invalid address record '{line}'."); - currentAddr = addr; - if (!segments.ContainsKey(addr)) segments[addr] = []; + currentData = []; + sections.Add((parsed, currentData)); continue; } - if (currentAddr is null) + + if (currentData is null) throw new InvalidDataException($"TiTxt: data before first '@address' line ('{line}')."); dataLines++; foreach (var token in line.Split([' ', '\t'], StringSplitOptions.RemoveEmptyEntries)) { - if (!byte.TryParse(token, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var b)) + if (token.Length != 2 || !byte.TryParse(token, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var value)) throw new InvalidDataException($"TiTxt: invalid hex byte '{token}'."); - segments[currentAddr.Value].Add(b); + currentData.Add(value); } } if (!sawTerminator) throw new InvalidDataException("TiTxt: missing 'q' termination line."); - var dict = new SortedDictionary(); - foreach (var (k, v) in segments) dict[k] = v.ToArray(); - var (merged, gapCount, totalBytes) = IntelHexReader.MergeSegments(dict); + var merged = NormalizeSections(sections, out var gapCount, out var totalBytes); return new FirmwareImage(merged, StartAddress: null, RecordCount: dataLines, GapCount: gapCount, TotalDataBytes: totalBytes, SourceFormat: "TiTxt"); } + + private static List<(uint Address, byte[] Data)> NormalizeSections( + List<(uint Address, List Data)> sections, out int gapCount, out int totalBytes) { + var result = new List<(uint Address, byte[] Data)>(); + gapCount = 0; + totalBytes = 0; + ulong previousEnd = 0; + var havePrevious = false; + + foreach (var (address, bytes) in sections.Where(section => section.Data.Count != 0).OrderBy(section => section.Address)) { + var end = (ulong)address + (ulong)bytes.Count; + if (end > (ulong)uint.MaxValue + 1) + throw new InvalidDataException($"TiTxt: section at 0x{address:X} extends beyond the 32-bit address space."); + if (havePrevious && address < previousEnd) + throw new InvalidDataException($"TiTxt: section at 0x{address:X} overlaps a preceding section ending at 0x{previousEnd - 1:X}."); + + if (havePrevious && address == previousEnd) { + var previous = result[^1]; + var joined = new byte[previous.Data.Length + bytes.Count]; + previous.Data.CopyTo(joined, 0); + bytes.CopyTo(joined, previous.Data.Length); + result[^1] = (previous.Address, joined); + } else { + if (havePrevious) gapCount++; + result.Add((address, bytes.ToArray())); + } + + previousEnd = end; + havePrevious = true; + checked { totalBytes += bytes.Count; } + } + + return result; + } } diff --git a/FileFormats/FileFormat.Ipsw/IpswFormatDescriptor.cs b/FileFormats/FileFormat.Ipsw/IpswFormatDescriptor.cs index 9b7fba649..72c956edc 100644 --- a/FileFormats/FileFormat.Ipsw/IpswFormatDescriptor.cs +++ b/FileFormats/FileFormat.Ipsw/IpswFormatDescriptor.cs @@ -3,316 +3,433 @@ using System.IO.Compression; using System.Text; using Compression.Registry; +using Compression.Registry.Streaming; using static Compression.Registry.FormatHelpers; namespace FileFormat.Ipsw; /// -/// Apple IPSW / OTA firmware package. An IPSW is just a ZIP file (with an Apple-specific layout). -/// Rather than surfacing entries as a flat generic ZIP, this descriptor lifts the well-known -/// Apple artifacts (BuildManifest.plist, Firmware/ subtree, LLB.*, iBSS.*, -/// iBEC.*, iBoot.*, root-filesystem *.dmg) into first-class canonical entries. -/// Everything else is exposed under other/. -/// -/// This is a compound-extension descriptor (.ipsw, .otazip): magic is empty so -/// it does not steal generic ZIPs. Read-only; the plist and DMG payloads are emitted as raw bytes -/// — no plist parsing or DMG mounting. -/// -/// References: -/// -/// https://theapplewiki.com — The Apple Wiki (formerly The iPhone Wiki) — community IPSW documentation -/// https://github.com/blacktop/ipsw — ipsw — maintained IPSW research and extraction tool -/// https://en.wikipedia.org/wiki/IPSW — Wikipedia -/// +/// Apple IPSW / OTA firmware package. IPSW files are ZIP containers whose entry paths are +/// semantically significant: manifests refer to firmware members by their ZIP path, so the +/// descriptor exposes those paths losslessly instead of projecting them into a flattened view. /// -public sealed class IpswFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveLayoutMap, IArchiveCreatable, IArchiveModifiable { +/// +/// The ZIP container itself is mutable. Inner DMG, IMG4/IM4P and other firmware payloads +/// remain opaque byte streams and are handled by their own descriptors when opened separately. +/// Rebuild maintenance (defrag/shrink) re-encodes only the ZIP envelope. Entry bytes and +/// entry paths are preserved; Apple restore tooling also accepts an extracted IPSW directory, +/// so ZIP physical offsets are not part of the firmware payload identity. +/// FULL.ipsw and metadata.ini are synthetic views reserved by this descriptor +/// and are never written back as ZIP members. +/// +public sealed class IpswFormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IArchiveLayoutMap, + IArchiveCreatable, + IArchiveModifiable, + IArchiveDefragmentable, + IArchiveShrinkable, + ISyntheticEntryNames { - /// - public IEnumerable EnumerateLayout(Stream archive) => FileFormat.Zip.ZipLayoutMap.Enumerate(archive); + private static readonly HashSet SyntheticEntries = new(StringComparer.OrdinalIgnoreCase) { + "FULL.ipsw", + "metadata.ini", + }; + + private sealed record IpswEntry( + string Name, + long Size, + long CompressedSize, + string Method, + bool IsDirectory, + DateTime? LastModified, + string Kind); + + public IReadOnlySet SyntheticEntryNames => SyntheticEntries; + + public IEnumerable EnumerateLayout(Stream archive) + => FileFormat.Zip.ZipLayoutMap.Enumerate(archive); - /// - /// Gets the id. - /// public string Id => "Ipsw"; - /// - /// Gets the display name. - /// public string DisplayName => "Apple IPSW"; - /// - /// Gets the category. - /// public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; - /// - /// Gets the default extension. - /// public string DefaultExtension => ".ipsw"; - /// - /// Gets the extensions. - /// public IReadOnlyList Extensions => []; - /// - /// Gets the compound extensions. - /// public IReadOnlyList CompoundExtensions => [".ipsw", ".otazip"]; - /// - /// Gets the magic signatures. - /// public IReadOnlyList MagicSignatures => []; - /// - /// Gets the methods. - /// public IReadOnlyList Methods => [new("deflate", "Deflate"), new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// public string Description => - "Apple firmware package (ZIP containing BuildManifest.plist, Firmware/, DMG root FS). " + - "R/W: in-place Add / Remove against the ZIP central directory (delegates to ZipModifier). " + - "Inner DMG / firmware blob mutation is delegated to FileFormat.Dmg and the per-firmware descriptors."; + "Apple firmware package (ZIP containing BuildManifest.plist, Firmware/, boot firmware and disk images). " + + "R/W edits operate on the ZIP namespace; defrag/shrink rebuild the ZIP envelope without changing payload bytes."; - private sealed record CanonicalEntry(string CanonicalName, string ZipEntryName, long Size, string Method, DateTime? LastModified, string? Kind); - - /// - /// Lists the entries in the supplied container. - /// public List List(Stream stream, string? password) { - var (canonical, total) = EnumerateCanonical(stream); - var entries = new List(2 + canonical.Count); - entries.Add(new ArchiveEntryInfo(0, "FULL.ipsw", stream.Length, stream.Length, "Stored", false, false, null)); - entries.Add(new ArchiveEntryInfo(1, "metadata.ini", 0, 0, "Stored", false, false, null, Kind: $"total_zip_entries={total}")); - for (var i = 0; i < canonical.Count; ++i) { - var c = canonical[i]; - entries.Add(new ArchiveEntryInfo( - Index: 2 + i, - Name: c.CanonicalName, - OriginalSize: c.Size, - CompressedSize: c.Size, - Method: c.Method, - IsDirectory: false, - IsEncrypted: false, - LastModified: c.LastModified, - Kind: c.Kind)); + ArgumentNullException.ThrowIfNull(stream); + var entries = EnumerateEntries(stream); + var result = new List(entries.Count + SyntheticEntries.Count) { + new(0, "FULL.ipsw", stream.Length, stream.Length, "Stored", false, false, null, Kind: "container"), + new(1, "metadata.ini", 0, 0, "Stored", false, false, null, Kind: $"total_zip_entries={entries.Count}"), + }; + + foreach (var entry in entries) { + if (IsSynthetic(entry.Name)) + continue; + result.Add(new ArchiveEntryInfo( + result.Count, + entry.Name, + entry.Size, + entry.CompressedSize, + entry.Method, + entry.IsDirectory, + false, + entry.LastModified, + Kind: entry.Kind)); } - return entries; + + return result; } - /// - /// Decodes the supplied input. - /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - // FULL.ipsw: stream directly from the input — never materialize into memory. + ArgumentNullException.ThrowIfNull(stream); + ArgumentException.ThrowIfNullOrWhiteSpace(outputDir); + if (Wants(files, "FULL.ipsw")) { - stream.Seek(0, SeekOrigin.Begin); - var fullPath = Path.Combine(outputDir, "FULL.ipsw"); - var dir = Path.GetDirectoryName(fullPath); - if (dir != null) Directory.CreateDirectory(dir); - using var outStream = File.Create(fullPath); - stream.CopyTo(outStream); + stream.Position = 0; + var fullPath = SafeCombine(outputDir, "FULL.ipsw"); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + using var output = File.Create(fullPath); + stream.CopyTo(output); } - // Re-open ZIP on the seekable input stream. - stream.Seek(0, SeekOrigin.Begin); + stream.Position = 0; using var zip = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); - var (canonical, total) = EnumerateCanonicalFromZip(zip); - - string? identifier = null; - string? productVersion = null; - string? buildVersion = null; + foreach (var entry in zip.Entries) { + var name = NormalizeZipPath(entry.FullName); + if (IsSynthetic(name) || !Wants(files, name)) + continue; - foreach (var c in canonical) { - if (!Wants(files, c.CanonicalName)) continue; - var zipEntry = zip.GetEntry(c.ZipEntryName); - if (zipEntry == null) continue; - - var destPath = SafeCombine(outputDir, c.CanonicalName); - var destDir = Path.GetDirectoryName(destPath); - if (destDir != null) Directory.CreateDirectory(destDir); - using (var es = zipEntry.Open()) - using (var outFile = File.Create(destPath)) { - es.CopyTo(outFile); + var destination = SafeCombine(outputDir, name); + if (IsDirectoryEntry(entry)) { + Directory.CreateDirectory(destination); + continue; } - if (c.CanonicalName == "BuildManifest.plist") { - // BuildManifest is bounded (typically <1 MB) — safe to read back for metadata parsing. - var data = File.ReadAllBytes(destPath); - TryParsePlistFields(data, out identifier, out productVersion, out buildVersion); - } + var parent = Path.GetDirectoryName(destination); + if (!string.IsNullOrEmpty(parent)) + Directory.CreateDirectory(parent); + using var source = entry.Open(); + using var output = File.Create(destination); + source.CopyTo(output); } if (Wants(files, "metadata.ini")) { - // If we didn't extract the manifest above, best-effort parse it now to populate metadata. - if (identifier == null && productVersion == null && buildVersion == null) { - var manifest = zip.GetEntry("BuildManifest.plist"); - if (manifest != null) { - // Manifest is small and bounded — fine to materialize. - using var es = manifest.Open(); - using var ms = new MemoryStream(); - es.CopyTo(ms); - TryParsePlistFields(ms.ToArray(), out identifier, out productVersion, out buildVersion); - } - } - WriteFile(outputDir, "metadata.ini", - Encoding.UTF8.GetBytes(BuildMetadataIni(identifier, productVersion, buildVersion, total))); + var metadata = BuildMetadata(zip); + var metadataPath = SafeCombine(outputDir, "metadata.ini"); + Directory.CreateDirectory(Path.GetDirectoryName(metadataPath)!); + File.WriteAllBytes(metadataPath, metadata); } } - // ── IArchiveCreatable ───────────────────────────────────────────── + public Stream OpenEntry(Stream archive, string entryName, string? password) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentException.ThrowIfNullOrWhiteSpace(entryName); + + archive.Position = 0; + if (string.Equals(entryName, "FULL.ipsw", StringComparison.OrdinalIgnoreCase)) + return new BoundedEntryStream(archive, archive.Length, leaveOpen: true); + + if (string.Equals(entryName, "metadata.ini", StringComparison.OrdinalIgnoreCase)) { + using var zip = new ZipArchive(archive, ZipArchiveMode.Read, leaveOpen: true); + var metadata = BuildMetadata(zip); + return new BoundedEntryStream(new MemoryStream(metadata, writable: false), metadata.Length, leaveOpen: false); + } + + var reader = new ZipArchive(archive, ZipArchiveMode.Read, leaveOpen: true); + var normalized = NormalizeZipPath(entryName); + var entry = reader.Entries.FirstOrDefault(candidate => + string.Equals(NormalizeZipPath(candidate.FullName), normalized, StringComparison.OrdinalIgnoreCase)); + if (entry == null || IsDirectoryEntry(entry)) { + reader.Dispose(); + return new BoundedEntryStream(new MemoryStream([], writable: false), 0, leaveOpen: false); + } + + var owned = new ZipOwnedReadStream(entry.Open(), reader); + return new BoundedEntryStream(owned, entry.Length, leaveOpen: false); + } - /// - /// Emits a fresh IPSW (ZIP) container from the supplied inputs. Synthetic - /// canonical entries the descriptor surfaces on read (FULL.ipsw, - /// metadata.ini) are silently dropped — they aren't real ZIP entries. - /// All other inputs are stored under their . - /// public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { ArgumentNullException.ThrowIfNull(output); ArgumentNullException.ThrowIfNull(inputs); - using var zip = new FileFormat.Zip.ZipWriter(output, leaveOpen: true); - foreach (var (name, data) in FilesOnly(inputs)) { - if (string.Equals(name, "FULL.ipsw", StringComparison.OrdinalIgnoreCase)) continue; - if (string.Equals(name, "metadata.ini", StringComparison.OrdinalIgnoreCase)) continue; - zip.AddEntry(name, data); + if (!output.CanWrite) + throw new ArgumentException("IPSW creation requires a writable output stream.", nameof(output)); + + if (output.CanSeek) { + output.Position = 0; + output.SetLength(0); } - zip.Finish(); - } - // ── IArchiveModifiable ───────────────────────────────────────────── + using var zip = new ZipArchive(output, ZipArchiveMode.Create, leaveOpen: true); + foreach (var input in inputs) { + var name = NormalizeZipPath(input.ArchiveName); + if (IsSynthetic(name)) + continue; + + if (input.IsDirectory) { + if (!name.EndsWith('/')) + name += '/'; + zip.CreateEntry(name, CompressionLevel.NoCompression); + continue; + } + + var size = input.InMemoryContent?.LongLength ?? new FileInfo(input.FullPath).Length; + var entry = zip.CreateEntry(name, SelectCompression(name, size)); + TryApplyTimestamp(entry, input); + using var destination = entry.Open(); + if (input.InMemoryContent is { } bytes) { + destination.Write(bytes); + } else { + using var source = File.OpenRead(input.FullPath); + source.CopyTo(destination); + } + } + } - /// - /// Adds (or replaces by ZIP path) entries inside an existing IPSW. - /// Routes through — only the central - /// directory, EOCD, and the appended LFH + payload are touched. - /// Synthetic canonical entries are silently dropped. - /// public void Add(Stream archive, IReadOnlyList inputs) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(inputs); - foreach (var (name, data) in FilesOnly(inputs)) { - if (string.Equals(name, "FULL.ipsw", StringComparison.OrdinalIgnoreCase)) continue; - if (string.Equals(name, "metadata.ini", StringComparison.OrdinalIgnoreCase)) continue; - IpswInPlaceModifier.AddEntry(archive, name, data); + foreach (var input in inputs) { + if (input.IsDirectory) + continue; + var name = NormalizeZipPath(input.ArchiveName); + if (IsSynthetic(name)) + continue; + IpswInPlaceModifier.AddEntry(archive, name, input.ReadContent()); } } - /// - /// Removes named ZIP entries from an existing IPSW. Routes through - /// — the LFH + compressed payload of - /// the dropped entry are zero-wiped and the central directory is rewritten. - /// public void Remove(Stream archive, string[] entryNames) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(entryNames); foreach (var name in entryNames) { - if (string.Equals(name, "FULL.ipsw", StringComparison.OrdinalIgnoreCase)) continue; - if (string.Equals(name, "metadata.ini", StringComparison.OrdinalIgnoreCase)) continue; - IpswInPlaceModifier.RemoveEntry(archive, name); + var normalized = NormalizeZipPath(name); + if (IsSynthetic(normalized)) + continue; + IpswInPlaceModifier.RemoveEntry(archive, normalized); } } - private static bool Wants(string[]? files, string name) - => files == null || files.Length == 0 || MatchesFilter(name, files); + public void Defragment(Stream archive) + => this.Defragment(archive, new DefragOptions { Mode = DefragMode.ConsolidateAtStart }); - private static (List Canonical, int TotalZipEntries) EnumerateCanonical(Stream stream) { - stream.Seek(0, SeekOrigin.Begin); - using var zip = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); - return EnumerateCanonicalFromZip(zip); + public void Defragment(Stream archive, DefragOptions options) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(options); + if (options.Mode != DefragMode.ConsolidateAtStart) + throw new NotSupportedException($"IPSW defrag supports only {DefragMode.ConsolidateAtStart}."); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("IPSW defrag requires a readable, writable, seekable stream.", nameof(archive)); + + using var rebuilt = CreateScratchStream(); + RebuildVerb.RebuildToStream( + archive, + rebuilt, + this, + this, + syntheticNames: SyntheticEntries, + onProgress: options.OnProgress, + cancellationToken: options.CancellationToken); + + options.CancellationToken.ThrowIfCancellationRequested(); + options.OnProgress?.Invoke(new DefragProgressEvent( + "committing", 0.99, -1, 0, Math.Max(1, rebuilt.Length), null, + "Committing verified IPSW rebuild")); + + archive.Position = 0; + archive.SetLength(0); + rebuilt.Position = 0; + rebuilt.CopyTo(archive); + archive.Flush(); + + archive.Position = 0; + var finalLayout = this.EnumerateLayout(archive).ToArray(); + options.OnProgress?.Invoke(new DefragProgressEvent( + "complete", 1, -1, -1, Math.Max(1, archive.Length), finalLayout, + "IPSW rebuild committed successfully")); } - private static (List Canonical, int TotalZipEntries) EnumerateCanonicalFromZip(ZipArchive zip) { - var canonical = new List(); - foreach (var entry in zip.Entries) { - // Skip directory entries (trailing slash, zero length implied). - if (entry.FullName.EndsWith('/')) continue; + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("IPSW shrink requires a readable, seekable input stream.", nameof(input)); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("IPSW shrink requires a writable, seekable output stream.", nameof(output)); - var name = entry.FullName.Replace('\\', '/'); - var filename = Path.GetFileName(name); - var method = entry.CompressedLength == entry.Length ? "Stored" : "Deflate"; - DateTime? lastModified = entry.LastWriteTime.DateTime; - - string canonicalName; - string? kind; - if (string.Equals(filename, "BuildManifest.plist", StringComparison.OrdinalIgnoreCase)) { - canonicalName = "BuildManifest.plist"; - kind = "manifest"; - } else if (name.Contains("Firmware/", StringComparison.OrdinalIgnoreCase)) { - canonicalName = "Firmware/" + filename; - kind = "firmware"; - } else if (IsBootloaderStage(filename)) { - canonicalName = filename; - kind = "bootloader"; - } else if (filename.EndsWith(".dmg", StringComparison.OrdinalIgnoreCase)) { - canonicalName = filename; - kind = "rootfs"; - } else { - canonicalName = "other/" + name; - kind = "other"; + using var rebuilt = CreateScratchStream(); + var useRebuilt = false; + try { + RebuildVerb.RebuildToStream(input, rebuilt, this, this, syntheticNames: SyntheticEntries); + useRebuilt = rebuilt.Length > 0 && rebuilt.Length < input.Length; + } catch { + useRebuilt = false; + } + + if (ReferenceEquals(input, output)) { + if (!useRebuilt) { + input.Position = 0; + return; } + output.Position = 0; + output.SetLength(0); + rebuilt.Position = 0; + rebuilt.CopyTo(output); + output.Flush(); + output.Position = 0; + return; + } + + output.Position = 0; + output.SetLength(0); + if (useRebuilt) { + rebuilt.Position = 0; + rebuilt.CopyTo(output); + } else { + input.Position = 0; + input.CopyTo(output); + } + output.Flush(); + output.Position = 0; + } + + private static List EnumerateEntries(Stream stream) { + stream.Position = 0; + using var zip = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); + var result = new List(zip.Entries.Count); + foreach (var entry in zip.Entries) { + var name = NormalizeZipPath(entry.FullName); + var isDirectory = IsDirectoryEntry(entry); + var method = entry.CompressedLength == entry.Length ? "Stored" : "Deflate"; + result.Add(new IpswEntry( + name, + entry.Length, + entry.CompressedLength, + method, + isDirectory, + entry.LastWriteTime.DateTime, + ClassifyKind(name, isDirectory))); + } + return result; + } + + private static byte[] BuildMetadata(ZipArchive zip) { + string? identifier = null; + string? productVersion = null; + string? buildVersion = null; + var manifest = zip.Entries.FirstOrDefault(entry => + string.Equals(NormalizeZipPath(entry.FullName), "BuildManifest.plist", StringComparison.OrdinalIgnoreCase)); + if (manifest != null && manifest.Length <= 16 * 1024 * 1024) { + using var source = manifest.Open(); + using var memory = new MemoryStream(); + source.CopyTo(memory); + TryParsePlistFields(memory.ToArray(), out identifier, out productVersion, out buildVersion); + } + return Encoding.UTF8.GetBytes(BuildMetadataIni(identifier, productVersion, buildVersion, zip.Entries.Count)); + } + + private static string ClassifyKind(string name, bool isDirectory) { + if (isDirectory) + return "directory"; + var fileName = LeafName(name); + if (string.Equals(name, "BuildManifest.plist", StringComparison.OrdinalIgnoreCase)) + return "manifest"; + if (string.Equals(name, "Restore.plist", StringComparison.OrdinalIgnoreCase) || + string.Equals(name, "RestoreVersion.plist", StringComparison.OrdinalIgnoreCase)) + return "restore-metadata"; + if (name.StartsWith("Firmware/", StringComparison.OrdinalIgnoreCase)) + return "firmware"; + if (IsBootloaderStage(fileName)) + return "bootloader"; + if (fileName.EndsWith(".dmg", StringComparison.OrdinalIgnoreCase)) + return "disk-image"; + return "other"; + } + + private static CompressionLevel SelectCompression(string name, long size) { + var fileName = LeafName(name); + if (size >= 32L * 1024 * 1024 || + fileName.EndsWith(".dmg", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".img4", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".im4p", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".im4m", StringComparison.OrdinalIgnoreCase)) + return CompressionLevel.NoCompression; + return CompressionLevel.Optimal; + } - // Use entry.Length — never call entry.Open() during List enumeration. - canonical.Add(new CanonicalEntry( - CanonicalName: canonicalName, - ZipEntryName: entry.FullName, - Size: entry.Length, - Method: method, - LastModified: lastModified, - Kind: kind)); + private static void TryApplyTimestamp(ZipArchiveEntry entry, ArchiveInputInfo input) { + if (input.InMemoryContent != null || !File.Exists(input.FullPath)) + return; + try { + entry.LastWriteTime = File.GetLastWriteTime(input.FullPath); + } catch (ArgumentOutOfRangeException) { + // ZIP/DOS timestamps are bounded to 1980..2107; payload data is unaffected. } - var total = zip.Entries.Count; - return (canonical, total); } - private static bool IsBootloaderStage(string filename) { - // Apple boot stage prefixes: LLB., iBSS., iBEC., iBoot. - if (filename.StartsWith("LLB.", StringComparison.OrdinalIgnoreCase)) return true; - if (filename.StartsWith("iBSS.", StringComparison.OrdinalIgnoreCase)) return true; - if (filename.StartsWith("iBEC.", StringComparison.OrdinalIgnoreCase)) return true; - if (filename.StartsWith("iBoot.", StringComparison.OrdinalIgnoreCase)) return true; - return false; + private static bool Wants(string[]? files, string name) + => files == null || files.Length == 0 || MatchesFilter(name, files); + + private static bool IsSynthetic(string name) + => SyntheticEntries.Contains(name); + + private static bool IsDirectoryEntry(ZipArchiveEntry entry) + => entry.FullName.EndsWith('/') || entry.FullName.EndsWith('\\'); + + private static string NormalizeZipPath(string name) + => name.Replace('\\', '/').TrimStart('/'); + + private static string LeafName(string name) { + var normalized = NormalizeZipPath(name).TrimEnd('/'); + var slash = normalized.LastIndexOf('/'); + return slash < 0 ? normalized : normalized[(slash + 1)..]; } - /// - /// Mirrors the path-sanitization done by so we can - /// stream directly into the target file without first materializing the payload. - /// + private static bool IsBootloaderStage(string filename) + => filename.StartsWith("LLB.", StringComparison.OrdinalIgnoreCase) || + filename.StartsWith("iBSS.", StringComparison.OrdinalIgnoreCase) || + filename.StartsWith("iBEC.", StringComparison.OrdinalIgnoreCase) || + filename.StartsWith("iBoot.", StringComparison.OrdinalIgnoreCase); + private static string SafeCombine(string baseDir, string entryName) { - var safeName = entryName.Replace('\\', '/').TrimStart('/'); - if (safeName.Contains("..")) safeName = Path.GetFileName(safeName); - return Path.Combine(baseDir, safeName); + var normalized = NormalizeZipPath(entryName); + var segments = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Any(segment => segment is "." or "..")) + throw new InvalidDataException($"IPSW entry path escapes extraction root: {entryName}"); + + var root = Path.GetFullPath(baseDir); + var candidate = Path.GetFullPath(Path.Combine(root, Path.Combine(segments))); + var rootPrefix = root.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; + if (!candidate.StartsWith(rootPrefix, StringComparison.OrdinalIgnoreCase) && + !string.Equals(candidate, root, StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException($"IPSW entry path escapes extraction root: {entryName}"); + return candidate; } - /// - /// Best-effort plist field scrape. We intentionally avoid full plist parsing; this just finds - /// the <key>...</key><string>...</string> pairs for a few known Apple - /// manifest fields (ProductVersion, ProductBuildVersion, identifier-like). Returns null if the - /// plist is binary or un-parseable. - /// private static void TryParsePlistFields(byte[] data, out string? identifier, out string? productVersion, out string? buildVersion) { identifier = null; productVersion = null; buildVersion = null; - if (data.Length < 16) return; - // Binary plist — give up; we surface raw bytes so consumers can parse. - if (data[0] == (byte)'b' && data[1] == (byte)'p' && data[2] == (byte)'l') return; - - string text; - try { text = Encoding.UTF8.GetString(data); } - catch { return; } + if (data.Length < 16) + return; + if (data.AsSpan().StartsWith("bplist"u8)) + return; + var text = Encoding.UTF8.GetString(data); productVersion = FindStringValue(text, "ProductVersion"); buildVersion = FindStringValue(text, "ProductBuildVersion") ?? FindStringValue(text, "BuildVersion"); identifier = FindStringValue(text, "ProductType") ?? FindStringValue(text, "Identifier"); @@ -320,23 +437,55 @@ private static void TryParsePlistFields(byte[] data, out string? identifier, out private static string? FindStringValue(string plistXml, string key) { var keyTag = $"{key}"; - var keyIdx = plistXml.IndexOf(keyTag, StringComparison.Ordinal); - if (keyIdx < 0) return null; - var searchFrom = keyIdx + keyTag.Length; - var openIdx = plistXml.IndexOf("", searchFrom, StringComparison.Ordinal); - if (openIdx < 0) return null; - var closeIdx = plistXml.IndexOf("", openIdx + 8, StringComparison.Ordinal); - if (closeIdx < 0) return null; - return plistXml.Substring(openIdx + 8, closeIdx - (openIdx + 8)); + var keyIndex = plistXml.IndexOf(keyTag, StringComparison.Ordinal); + if (keyIndex < 0) + return null; + var openIndex = plistXml.IndexOf("", keyIndex + keyTag.Length, StringComparison.Ordinal); + if (openIndex < 0) + return null; + var closeIndex = plistXml.IndexOf("", openIndex + 8, StringComparison.Ordinal); + return closeIndex < 0 ? null : plistXml[(openIndex + 8)..closeIndex]; } private static string BuildMetadataIni(string? identifier, string? productVersion, string? buildVersion, int totalZipEntries) { - var sb = new StringBuilder(); - sb.Append("[Ipsw]\n"); - sb.Append(CultureInfo.InvariantCulture, $"identifier={identifier ?? string.Empty}\n"); - sb.Append(CultureInfo.InvariantCulture, $"product_version={productVersion ?? string.Empty}\n"); - sb.Append(CultureInfo.InvariantCulture, $"build_version={buildVersion ?? string.Empty}\n"); - sb.Append(CultureInfo.InvariantCulture, $"total_zip_entries={totalZipEntries}\n"); - return sb.ToString(); + var builder = new StringBuilder(); + builder.Append("[Ipsw]\n"); + builder.Append(CultureInfo.InvariantCulture, $"identifier={identifier ?? string.Empty}\n"); + builder.Append(CultureInfo.InvariantCulture, $"product_version={productVersion ?? string.Empty}\n"); + builder.Append(CultureInfo.InvariantCulture, $"build_version={buildVersion ?? string.Empty}\n"); + builder.Append(CultureInfo.InvariantCulture, $"total_zip_entries={totalZipEntries}\n"); + return builder.ToString(); + } + + private static FileStream CreateScratchStream() + => new( + Path.Combine(Path.GetTempPath(), "cwb_ipsw_" + Guid.NewGuid().ToString("N") + ".tmp"), + FileMode.CreateNew, + FileAccess.ReadWrite, + FileShare.None, + 64 * 1024, + FileOptions.DeleteOnClose); + + private sealed class ZipOwnedReadStream(Stream inner, ZipArchive owner) : Stream { + public override bool CanRead => inner.CanRead; + public override bool CanSeek => inner.CanSeek; + public override bool CanWrite => false; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => inner.Position = value; } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); + public override int Read(Span buffer) => inner.Read(buffer); + public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override void Write(ReadOnlySpan buffer) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) { + if (disposing) { + inner.Dispose(); + owner.Dispose(); + } + base.Dispose(disposing); + } } } diff --git a/FileFormats/FileFormat.Macrium/MacriumFormatDescriptor.cs b/FileFormats/FileFormat.Macrium/MacriumFormatDescriptor.cs index c6faf2b27..1d5a148d8 100644 --- a/FileFormats/FileFormat.Macrium/MacriumFormatDescriptor.cs +++ b/FileFormats/FileFormat.Macrium/MacriumFormatDescriptor.cs @@ -36,8 +36,8 @@ namespace FileFormat.Macrium; /// /// R/W semantics: Macrium Reflect X is a disk-image format whose /// logical payload is a single contiguous sector stream — the same shape as -/// VHD / VDI / VMDK / QCOW2. and -/// operate on the synthetic +/// VHD / VDI / VMDK / QCOW2. and +/// operate on the synthetic /// disk-image.raw entry (Add concatenates supplied input bytes onto /// the existing image and rebuilds the container; Add of an entry whose name /// matches an existing one replaces the disk payload; Remove of diff --git a/FileFormats/FileFormat.T64/LynxFormatDescriptor.cs b/FileFormats/FileFormat.T64/LynxFormatDescriptor.cs index 88fa95875..d00617987 100644 --- a/FileFormats/FileFormat.T64/LynxFormatDescriptor.cs +++ b/FileFormats/FileFormat.T64/LynxFormatDescriptor.cs @@ -16,7 +16,10 @@ public sealed class LynxFormatDescriptor : IArchiveCreatable, IArchiveModifiable, IArchiveDefragmentable, + IArchiveShrinkable, IArchiveLayoutMap, + IWipeEmpty, + IArchivePurgeable, IFormatOptionsSchema { /// @@ -76,8 +79,8 @@ public sealed class LynxFormatDescriptor : /// public string Description => "Will Corley/Ultimate Lynx Commodore archive: 254-byte sector payload blocks, PRG/SEQ/USR/DEL " + - "creation, REL read/remove support, and genuine in-place add/replace/remove by shifting only " + - "the affected directory/data block ranges. No compression or checksum exists in the format."; + "creation, REL read/remove support, genuine in-place add/replace/remove, exact layout/wipe, " + + "directory/trailer shrink, and purge. No compression or checksum exists in the format."; /// /// Gets the options schema. @@ -183,6 +186,25 @@ public void Remove(Stream archive, string[] entryNames) { LynxInPlaceModifier.Remove(archive, name); } + /// + /// Removes all live entries in one pass while preserving the input archive's BASIC preamble + /// and Lynx signature. The empty directory is emitted at its minimum one-block allocation. + /// + public void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("Lynx purge requires a readable, writable, seekable stream.", nameof(archive)); + + var reader = Open(archive); + var directory = LynxWriter.BuildDirectory([], 1, reader.BasicHeader, reader.Signature); + archive.Position = 0; + archive.SetLength(0); + archive.Write(directory); + archive.SetLength(directory.Length); + archive.Flush(); + archive.Position = 0; + } + /// /// Lynx data extents are inherently contiguous and ordered by the directory. Defragmentation /// therefore consists of validating that layout and dropping transport/trailing padding after @@ -210,18 +232,109 @@ public void Defragment(Stream archive, DefragOptions options) { } /// - /// Enumerates the layout. + /// Rebuilds the directory at its smallest whole-254-byte allocation and copies the complete + /// existing data area byte-for-byte. This preserves REL side sectors and file-type metadata, + /// while reclaiming directory blocks left behind after removals and any trailing transport data. + /// + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (ReferenceEquals(input, output)) + throw new ArgumentException("Lynx shrink requires distinct input and output streams.", nameof(output)); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("Lynx shrink input must be readable and seekable.", nameof(input)); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("Lynx shrink output must be writable and seekable.", nameof(output)); + + var reader = Open(input); + var directory = LynxWriter.BuildDirectory( + reader.Entries.Select(LynxWriter.FromEntry).ToArray(), + 1, + reader.BasicHeader, + reader.Signature); + + output.Position = 0; + output.SetLength(0); + output.Write(directory); + + input.Position = reader.DataStart; + var remaining = reader.LogicalDataEnd - reader.DataStart; + var buffer = new byte[64 * 1024]; + while (remaining > 0) { + var count = (int)Math.Min(buffer.Length, remaining); + input.ReadExactly(buffer.AsSpan(0, count)); + output.Write(buffer.AsSpan(0, count)); + remaining -= count; + } + + output.SetLength(output.Position); + output.Flush(); + output.Position = 0; + } + + /// + /// Enumerates the exact byte layout: live directory text, directory padding, REL side sectors, + /// logical file bytes, per-file block padding, and any trailer beyond the archive allocation. + /// Explicit free extents make generic forensic wiping safe for this format. /// public IEnumerable EnumerateLayout(Stream archive) { var reader = Open(archive); - yield return new DefragBlockInfo(0, reader.DataStart, DefragBlockKind.MetadataReserved, "Lynx directory"); + + if (reader.DirectoryContentEnd > 0) + yield return new DefragBlockInfo(0, reader.DirectoryContentEnd, DefragBlockKind.MetadataReserved, "Lynx directory"); + if (reader.DirectoryContentEnd < reader.DataStart) + yield return new DefragBlockInfo( + reader.DirectoryContentEnd, + reader.DataStart - reader.DirectoryContentEnd, + DefragBlockKind.Free, + "Directory padding"); + foreach (var entry in reader.Entries) { - var allocated = checked((long)entry.ArchiveBlocks * LynxReader.BlockSize); - if (allocated > 0) - yield return new DefragBlockInfo(entry.AllocationOffset, allocated, DefragBlockKind.Used, entry.Name); + if (entry.DataOffset > entry.AllocationOffset) + yield return new DefragBlockInfo( + entry.AllocationOffset, + entry.DataOffset - entry.AllocationOffset, + DefragBlockKind.MetadataReserved, + $"{entry.Name} REL side sectors"); + + if (entry.Length > 0) + yield return new DefragBlockInfo(entry.DataOffset, entry.Length, DefragBlockKind.Used, entry.Name); + + var allocationEnd = checked(entry.AllocationOffset + (long)entry.ArchiveBlocks * LynxReader.BlockSize); + var payloadEnd = checked(entry.DataOffset + entry.Length); + if (payloadEnd < allocationEnd) + yield return new DefragBlockInfo( + payloadEnd, + allocationEnd - payloadEnd, + DefragBlockKind.Free, + $"{entry.Name} block padding"); } + if (reader.LogicalDataEnd < archive.Length) - yield return new DefragBlockInfo(reader.LogicalDataEnd, archive.Length - reader.LogicalDataEnd, DefragBlockKind.Free); + yield return new DefragBlockInfo( + reader.LogicalDataEnd, + archive.Length - reader.LogicalDataEnd, + DefragBlockKind.Free, + "Trailing transport data"); + } + + /// + /// Zeros only byte ranges that the Lynx layout proves unused: directory padding, payload block + /// padding and trailing transport data. Live file bytes and REL side-sector metadata are retained. + /// + public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("Lynx wipe requires a readable, writable, seekable stream.", nameof(image)); + + var imageSize = image.Length; + var layout = this.EnumerateLayout(image).ToArray(); + return UnusedSpaceWiper.WipeDeclaredFree( + image, + layout, + imageSize, + wipeClusterTips: false, + fileSizeLookup: null); } private static LynxReader Open(Stream stream) { diff --git a/FileFormats/FileFormat.T64/LynxReader.cs b/FileFormats/FileFormat.T64/LynxReader.cs index 4304f835b..8623f6561 100644 --- a/FileFormats/FileFormat.T64/LynxReader.cs +++ b/FileFormats/FileFormat.T64/LynxReader.cs @@ -27,6 +27,7 @@ internal sealed class LynxReader { public int FileCount { get; private set; } public string Signature { get; private set; } = string.Empty; public byte[] BasicHeader { get; private set; } = []; + public long DirectoryContentEnd { get; private set; } public long DataStart => (long)this.DirectoryBlocks * BlockSize; public long LogicalDataEnd { get; private set; } @@ -141,6 +142,7 @@ private void Parse() { throw new InvalidDataException($"Lynx entry '{name}' allocation extends beyond the archive."); } + this.DirectoryContentEnd = cursor; this.LogicalDataEnd = allocationOffset; } diff --git a/FileFormats/FileFormat.UefiFv/UefiFvConstants.cs b/FileFormats/FileFormat.UefiFv/UefiFvConstants.cs new file mode 100644 index 000000000..ee95b432e --- /dev/null +++ b/FileFormats/FileFormat.UefiFv/UefiFvConstants.cs @@ -0,0 +1,59 @@ +#pragma warning disable CS1591 + +namespace FileFormat.UefiFv; + +internal static class UefiFvConstants { + internal const uint ErasePolarityMask = 0x00000800; + internal const byte LargeFile = 0x01; + internal const byte Alignment2 = 0x02; + internal const byte Fixed = 0x04; + internal const byte Alignment = 0x38; + internal const byte HeaderConstruction = 0x01; + internal const byte HeaderValid = 0x02; + internal const byte DataValid = 0x04; + internal const byte MarkedForUpdate = 0x08; + internal const byte Deleted = 0x10; + internal const byte HeaderInvalid = 0x20; + internal static readonly Guid Ffs2Guid = Guid.Parse("8C8CE578-8A3D-4F1C-9935-896185C32DD3"); + internal static readonly Guid Ffs3Guid = Guid.Parse("5473C07A-3DCB-4DCA-BD6F-1E9689E7349A"); + internal static readonly Guid VolumeTopGuid = Guid.Parse("1BA0062E-C779-4582-8566-336AE8F78F09"); + internal static readonly Guid SignedContentsGuid = Guid.Parse("0F9D89E8-9259-4F76-A5AF-0C89E34023DF"); + + internal static byte EffectiveState(byte raw, bool erasePolarity) { + var value = (byte)((erasePolarity ? ~raw : raw) & 0x3F); + if ((value & HeaderInvalid) != 0) return HeaderInvalid; + if ((value & Deleted) != 0) return Deleted; + if ((value & MarkedForUpdate) != 0) return MarkedForUpdate; + if ((value & DataValid) != 0) return DataValid; + if ((value & HeaderValid) != 0) return HeaderValid; + return (value & HeaderConstruction) != 0 ? HeaderConstruction : (byte)0; + } + + internal static byte EncodeState(byte cumulativeState, bool erasePolarity) + => erasePolarity ? unchecked((byte)~cumulativeState) : cumulativeState; + + internal static int DataAlignment(byte attributes) { + var code = (attributes & Alignment) >> 3; + if ((attributes & Alignment2) != 0) return 1 << (17 + code); + return code switch { 0 => 1, 1 => 16, 2 => 128, 3 => 512, 4 => 1024, 5 => 4096, 6 => 32768, _ => 65536 }; + } + + internal static int Align8(int value) => checked((int)(((long)value + 7) & ~7L)); + internal static int AlignOffset(int start, int absolute) => checked(start + Align8(checked(absolute - start))); +} + +internal sealed record UefiFvSlot(int Offset, int HeaderSize, int Size, int Footprint, Guid Name, + byte Type, byte Attributes, byte RawState, byte State) { + public int DataOffset => checked(this.Offset + this.HeaderSize); + public int DataLength => checked(this.Size - this.HeaderSize); + public int DataEnd => checked(this.Offset + this.Size); + public int End => checked(this.Offset + this.Footprint); + public bool IsPad => this.Type == 0xF0; + public bool IsFixed => (this.Attributes & UefiFvConstants.Fixed) != 0; + public int DataAlignment => UefiFvConstants.DataAlignment(this.Attributes); +} + +internal sealed record UefiFvLayout(int Start, int End, int DataStart, Guid FileSystemGuid, uint Attributes, + bool ErasePolarity, byte EraseByte, int? UsedSizeOffset, bool IsSigned, IReadOnlyList Slots) { + public bool SupportsLargeFiles => this.FileSystemGuid == UefiFvConstants.Ffs3Guid; +} diff --git a/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs b/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs index f23aae820..260247d73 100644 --- a/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs +++ b/FileFormats/FileFormat.UefiFv/UefiFvFormatDescriptor.cs @@ -7,87 +7,41 @@ namespace FileFormat.UefiFv; /// -/// UEFI PI Firmware Volume (.fv/.fd) archive surface. FFS files are -/// exposed as {GUID}_{TYPE_TAG}.bin; standalone volumes can be created and -/// ordinary FFS2 records can be added/replaced/removed through erased free space. +/// UEFI PI Firmware Volume archive surface. Standard FFS2/FFS3 files are exposed as +/// {GUID}_{TYPE_TAG}.bin; mutable unsigned volumes support transactional edits, +/// erase-aware wiping, purge, and offline consolidation. /// -/// References: -/// -/// https://uefi.org/specifications — UEFI Platform Initialization (PI) Specification, Volume 3: Firmware Storage Design -/// https://github.com/LongSoft/UEFITool — UEFITool firmware-volume parser/editor -/// +/// References: UEFI PI Specification 1.10 Volume III (Firmware Storage) and EDK II +/// as an interoperability oracle. The implementation is independently written. /// public sealed class UefiFvFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, - IArchiveCreatable, IArchiveModifiable { + IArchiveCreatable, IArchiveModifiable, IArchiveDefragmentable, IArchiveLayoutMap, IArchivePurgeable { - /// - /// Gets the id. - /// public string Id => "UefiFv"; - /// - /// Gets the display name. - /// public string DisplayName => "UEFI Firmware Volume"; - /// - /// Gets the category. - /// public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; - /// - /// Gets the default extension. - /// public string DefaultExtension => ".fv"; - /// - /// Gets the extensions. - /// public IReadOnlyList Extensions => [".fv", ".fd"]; - /// - /// Gets the compound extensions. - /// public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// public IReadOnlyList MagicSignatures => [ - new([(byte)'_', (byte)'F', (byte)'V', (byte)'H'], - Offset: UefiFvReader.SignatureOffset, Confidence: 0.95), + new([(byte)'_', (byte)'F', (byte)'V', (byte)'H'], Offset: UefiFvReader.SignatureOffset, Confidence: 0.95), ]; - /// - /// Gets the methods. - /// public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// public string Description => - "UEFI PI Firmware Volume — create and offline R/W for ordinary FFS2 files in fixed-capacity volumes."; + "UEFI PI Firmware Volume — FFS2/FFS3 create and transactional offline R/W, purge, wipe and defrag."; - /// - /// Lists the entries in the supplied container. - /// public List List(Stream stream, string? password) => BuildEntries(stream).Select((e, i) => new ArchiveEntryInfo( Index: i, Name: e.Name, OriginalSize: e.Data.LongLength, CompressedSize: e.Data.LongLength, Method: e.Method, IsDirectory: false, IsEncrypted: false, LastModified: null)).ToList(); - /// - /// Decodes the supplied input. - /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { foreach (var e in BuildEntries(stream)) { if (files != null && files.Length > 0 && !MatchesFilter(e.Name, files)) continue; @@ -109,6 +63,25 @@ public void Remove(Stream archive, string[] entryNames) => UefiFvInPlaceModifier.Remove(archive, entryNames.Where(n => !string.Equals(n, "metadata.ini", StringComparison.OrdinalIgnoreCase)).ToArray()); + public void Defragment(Stream archive) + => UefiFvMaintenance.Defragment(archive, new DefragOptions { Mode = DefragMode.ConsolidateAtStart }); + + public void Defragment(Stream archive, DefragOptions options) + => UefiFvMaintenance.Defragment(archive, options); + + public IEnumerable EnumerateLayout(Stream archive) + => UefiFvLayoutMap.Enumerate(archive); + + /// + /// Restores unused bytes to the FV's declared erase value. For erase-polarity-one + /// flash this is 0xFF rather than zero; writing zero would turn free space into + /// apparently programmed data and make the firmware volume structurally invalid. + /// + public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) + => UefiFvLayoutMap.Wipe(image, wipeDeletedEntries); + + public void Purge(Stream archive) => UefiFvMaintenance.Purge(archive); + private static List<(string Name, byte[] Data, string Method)> BuildEntries(Stream stream) { if (stream.CanSeek) stream.Position = 0; using var ms = new MemoryStream(); @@ -117,9 +90,7 @@ public void Remove(Stream archive, string[] entryNames) var fvStart = UefiFvReader.FindFirst(data) ?? 0; var fv = UefiFvReader.Read(data, fvStart); - var entries = new List<(string, byte[], string)> { - ("metadata.ini", BuildMetadata(fv), "stored"), - }; + var entries = new List<(string, byte[], string)> { ("metadata.ini", BuildMetadata(fv), "stored") }; foreach (var f in fv.Files) { if (f.Type == 0xF0) continue; entries.Add((UefiFvWriter.EntryName(f.Name, f.Type), f.Contents, "stored")); @@ -134,20 +105,18 @@ private static byte[] BuildMetadata(UefiFvReader.FirmwareVolume fv) { sb.Append(CultureInfo.InvariantCulture, $"file_system_guid = {fv.Header.FileSystemGuid:D}\n"); sb.Append(CultureInfo.InvariantCulture, $"fv_length = {fv.Header.FvLength}\n"); sb.Append(CultureInfo.InvariantCulture, $"attributes = 0x{fv.Header.Attributes:X8}\n"); + sb.Append(CultureInfo.InvariantCulture, $"erase_byte = 0x{fv.Header.EraseByte:X2}\n"); sb.Append(CultureInfo.InvariantCulture, $"header_length = {fv.Header.HeaderLength}\n"); sb.Append(CultureInfo.InvariantCulture, $"checksum = 0x{fv.Header.Checksum:X4}\n"); sb.Append(CultureInfo.InvariantCulture, $"ext_header_offset = 0x{fv.Header.ExtHeaderOffset:X}\n"); sb.Append(CultureInfo.InvariantCulture, $"revision = {fv.Header.Revision}\n"); sb.Append(CultureInfo.InvariantCulture, $"file_count = {fv.Files.Count}\n"); - sb.AppendLine(); sb.AppendLine("[block_map]"); for (var i = 0; i < fv.Header.BlockMap.Count; i++) { - var (nb, bl) = fv.Header.BlockMap[i]; - sb.Append(CultureInfo.InvariantCulture, - $"block_{i} = {nb} blocks x {bl} bytes\n"); + var (blocks, length) = fv.Header.BlockMap[i]; + sb.Append(CultureInfo.InvariantCulture, $"block_{i} = {blocks} blocks x {length} bytes\n"); } - sb.AppendLine(); sb.AppendLine("[files]"); for (var i = 0; i < fv.Files.Count; i++) { diff --git a/FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs b/FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs index f9f0d6b8f..53c84bdc8 100644 --- a/FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs +++ b/FileFormats/FileFormat.UefiFv/UefiFvInPlaceModifier.cs @@ -4,46 +4,42 @@ namespace FileFormat.UefiFv; -/// -/// Offline random-access editor for ordinary FFS2 records in a firmware volume. -/// It reuses erased 0xFF ranges and never relocates unrelated FFS files. -/// +/// Transactional offline editor for standard PI FFS2/FFS3 firmware volumes. internal static class UefiFvInPlaceModifier { - private sealed record Slot(int Offset, int Length, Guid Guid, byte Type) { - public string Name => UefiFvWriter.EntryName(Guid, Type); - } - public static void Add(Stream archive, IReadOnlyList inputs) { ArgumentNullException.ThrowIfNull(inputs); var state = Open(archive); foreach (var (name, data) in FilesOnly(inputs)) { var identity = UefiFvWriter.IdentityFromName(name); - var existing = ScanSlots(state.Image, state.FvStart, state.FvEnd) - .FirstOrDefault(s => s.Guid == identity.Guid); - if (existing != null) - Erase(state, existing.Offset, existing.Length); + foreach (var old in UefiFvParser.LiveSlots(state.Volume, includePad: false) + .Where(slot => slot.Name == identity.Guid).ToList()) + Erase(state, old.Offset, old.Footprint); + Refresh(state); - var encoded = UefiFvWriter.BuildFfsFile(identity.Guid, identity.Type, data); + var encoded = UefiFvWriter.BuildFfsFile(identity.Guid, identity.Type, data, + state.Volume.FileSystemGuid, state.Volume.ErasePolarity); var footprint = UefiFvWriter.Align8(encoded.Length); - var offset = FindErasedRun(state.Image, state.DataStart, state.FvEnd, footprint); + var offset = FindErasedRun(state.Image, state.Volume.DataStart, state.Volume.End, + footprint, state.Volume.EraseByte); if (offset < 0) throw new IOException($"UEFI FV has no erased run large enough for '{name}' ({footprint} bytes)."); - - Write(state, offset, encoded); - if (footprint > encoded.Length) - Erase(state, offset + encoded.Length, footprint - encoded.Length); + encoded.CopyTo(state.Image, offset); + Refresh(state); } + Finish(state); } public static void Remove(Stream archive, string[] entryNames) { ArgumentNullException.ThrowIfNull(entryNames); var state = Open(archive); foreach (var name in entryNames) { - var slot = ScanSlots(state.Image, state.FvStart, state.FvEnd) - .FirstOrDefault(s => string.Equals(s.Name, name, StringComparison.OrdinalIgnoreCase)); - if (slot != null) - Erase(state, slot.Offset, slot.Length); + var slot = UefiFvParser.LiveSlots(state.Volume, includePad: false) + .FirstOrDefault(s => string.Equals(UefiFvWriter.EntryName(s.Name, s.Type), name, StringComparison.OrdinalIgnoreCase)); + if (slot == null) continue; + Erase(state, slot.Offset, slot.Footprint); + Refresh(state); } + Finish(state); } private static EditorState Open(Stream archive) { @@ -52,76 +48,47 @@ private static EditorState Open(Stream archive) { throw new ArgumentException("UEFI FV mutation requires a seekable read/write stream.", nameof(archive)); if (archive.Length > int.MaxValue) throw new NotSupportedException("The in-memory FV editor supports images up to 2 GiB."); - archive.Position = 0; var image = new byte[checked((int)archive.Length)]; archive.ReadExactly(image); - var fvStart = UefiFvReader.FindFirst(image) - ?? throw new InvalidDataException("UEFI FV header was not found."); - var fv = UefiFvReader.Read(image, fvStart); - var fvEnd = checked(fvStart + (int)fv.Header.FvLength); - if (fvEnd > image.Length) throw new InvalidDataException("UEFI FV extends past the image."); - var dataStart = Align8(checked(fvStart + fv.Header.HeaderLength)); - return new EditorState(archive, image, fvStart, dataStart, fvEnd); + var start = UefiFvReader.FindFirst(image) ?? throw new InvalidDataException("UEFI FV header was not found."); + var volume = UefiFvParser.Parse(image, start); + EnsureMutable(volume); + return new EditorState(archive, image, start, volume); } - private static List ScanSlots(byte[] image, int fvStart, int fvEnd) { - var fv = UefiFvReader.Read(image, fvStart); - var pos = Align8(checked(fvStart + fv.Header.HeaderLength)); - var result = new List(); - while (pos + UefiFvWriter.FfsHeaderLength <= fvEnd) { - // One quantum at a time, and only the quantum is tested — see the same - // walk in UefiFvReader: testing a whole header straddles the end of an - // erased gap and reads the next file's GUID as a length. - if (IsErased(image.AsSpan(pos, UefiFvWriter.Alignment))) { - pos += UefiFvWriter.Alignment; - continue; - } - - var size = image[pos + 20] | (image[pos + 21] << 8) | (image[pos + 22] << 16); - if (size < UefiFvWriter.FfsHeaderLength || pos + size > fvEnd) - throw new InvalidDataException($"Invalid FFS file header at FV offset 0x{pos - fvStart:X}."); - var guid = new Guid(image.AsSpan(pos, 16)); - var type = image[pos + 18]; - var footprint = UefiFvWriter.Align8(size); - result.Add(new Slot(pos, footprint, guid, type)); - pos += footprint; - } - return result; + private static void EnsureMutable(UefiFvLayout volume) { + if (volume.FileSystemGuid != UefiFvConstants.Ffs2Guid && volume.FileSystemGuid != UefiFvConstants.Ffs3Guid) + throw new NotSupportedException($"UEFI FV uses unsupported file-system GUID {volume.FileSystemGuid:D}."); + if (volume.IsSigned) + throw new NotSupportedException("Signed UEFI firmware volumes cannot be modified without re-signing the complete volume."); } - private static int FindErasedRun(byte[] image, int start, int end, int needed) { - for (var pos = Align8(start); pos + needed <= end; pos += UefiFvWriter.Alignment) { - if (IsErased(image.AsSpan(pos, needed))) return pos; - } - return -1; - } + private static void Refresh(EditorState state) + => state.Volume = UefiFvParser.Parse(state.Image, state.Start); - private static bool IsErased(ReadOnlySpan bytes) { - foreach (var b in bytes) - if (b != 0xFF) return false; - return true; + private static void Finish(EditorState state) { + var usedEnd = UefiFvParser.LiveSlots(state.Volume).Select(slot => slot.DataEnd) + .DefaultIfEmpty(state.Volume.DataStart).Max(); + UefiFvParser.WriteUsedSize(state.Image, state.Volume, usedEnd); + state.Archive.Position = 0; + state.Archive.Write(state.Image); + state.Archive.SetLength(state.Image.Length); } - private static void Write(EditorState state, int offset, ReadOnlySpan bytes) { - bytes.CopyTo(state.Image.AsSpan(offset, bytes.Length)); - state.Archive.Position = offset; - state.Archive.Write(bytes); + private static int FindErasedRun(byte[] image, int start, int end, int needed, byte eraseByte) { + for (var pos = UefiFvConstants.AlignOffset(start, start); pos + needed <= end; pos += UefiFvWriter.Alignment) + if (UefiFvParser.IsErased(image.AsSpan(pos, needed), eraseByte)) return pos; + return -1; } - private static void Erase(EditorState state, int offset, int length) { - state.Image.AsSpan(offset, length).Fill(0xFF); - state.Archive.Position = offset; - Span erased = stackalloc byte[1024]; - erased.Fill(0xFF); - var remaining = length; - while (remaining > 0) { - var take = Math.Min(remaining, erased.Length); - state.Archive.Write(erased[..take]); - remaining -= take; - } - } + private static void Erase(EditorState state, int offset, int length) + => state.Image.AsSpan(offset, length).Fill(state.Volume.EraseByte); - private static int Align8(int value) => (value + 7) & ~7; - private sealed record EditorState(Stream Archive, byte[] Image, int FvStart, int DataStart, int FvEnd); + private sealed class EditorState(Stream archive, byte[] image, int start, UefiFvLayout volume) { + public Stream Archive { get; } = archive; + public byte[] Image { get; } = image; + public int Start { get; } = start; + public UefiFvLayout Volume { get; set; } = volume; + } } diff --git a/FileFormats/FileFormat.UefiFv/UefiFvLayoutMap.cs b/FileFormats/FileFormat.UefiFv/UefiFvLayoutMap.cs new file mode 100644 index 000000000..2652ef314 --- /dev/null +++ b/FileFormats/FileFormat.UefiFv/UefiFvLayoutMap.cs @@ -0,0 +1,59 @@ +#pragma warning disable CS1591 +using Compression.Registry; + +namespace FileFormat.UefiFv; + +internal static class UefiFvLayoutMap { + internal static IEnumerable Enumerate(Stream archive) { + var (image, volume) = UefiFvMaintenance.Read(archive, writable: false); + return Build(image.Length, volume, includeDeletedAsFree: true); + } + + internal static long Wipe(Stream archive, bool wipeDeletedEntries) { + var (image, volume) = UefiFvMaintenance.Read(archive, writable: true); + UefiFvMaintenance.EnsureMutable(volume); + long written = 0; + foreach (var extent in Build(image.Length, volume, wipeDeletedEntries).Where(e => e.Kind == DefragBlockKind.Free)) { + image.AsSpan(checked((int)extent.Offset), checked((int)extent.Length)).Fill(volume.EraseByte); + written += extent.Length; + } + + var reparsed = UefiFvParser.Parse(image, volume.Start); + var usedEnd = UefiFvParser.LiveSlots(reparsed).Select(slot => slot.DataEnd).DefaultIfEmpty(reparsed.DataStart).Max(); + UefiFvParser.WriteUsedSize(image, reparsed, usedEnd); + UefiFvMaintenance.Commit(archive, image); + return written; + } + + private static List Build(int imageLength, UefiFvLayout volume, bool includeDeletedAsFree) { + var result = new List(); + if (volume.DataStart > 0) + result.Add(new DefragBlockInfo(0, volume.DataStart, DefragBlockKind.MetadataReserved, "FV header / prefix")); + + var cursor = volume.DataStart; + foreach (var slot in volume.Slots.OrderBy(slot => slot.Offset)) { + if (cursor < slot.Offset) + result.Add(new DefragBlockInfo(cursor, slot.Offset - cursor, DefragBlockKind.Free)); + + DefragBlockKind kind; + string? name = null; + if (UefiFvParser.IsLive(volume, slot)) { + kind = slot.IsPad ? DefragBlockKind.MetadataReserved : DefragBlockKind.Used; + name = slot.IsPad ? "FFS pad" : UefiFvWriter.EntryName(slot.Name, slot.Type); + } else if (includeDeletedAsFree && UefiFvParser.IsDiscardable(volume, slot)) { + kind = DefragBlockKind.Free; + } else { + kind = DefragBlockKind.MetadataReserved; + name = "Incomplete FFS record"; + } + result.Add(new DefragBlockInfo(slot.Offset, slot.Footprint, kind, name)); + cursor = slot.End; + } + + if (cursor < volume.End) + result.Add(new DefragBlockInfo(cursor, volume.End - cursor, DefragBlockKind.Free)); + if (volume.End < imageLength) + result.Add(new DefragBlockInfo(volume.End, imageLength - volume.End, DefragBlockKind.MetadataReserved, "FV suffix")); + return result; + } +} diff --git a/FileFormats/FileFormat.UefiFv/UefiFvMaintenance.cs b/FileFormats/FileFormat.UefiFv/UefiFvMaintenance.cs new file mode 100644 index 000000000..1ec81d8b6 --- /dev/null +++ b/FileFormats/FileFormat.UefiFv/UefiFvMaintenance.cs @@ -0,0 +1,104 @@ +#pragma warning disable CS1591 +using Compression.Registry; + +namespace FileFormat.UefiFv; + +internal static class UefiFvMaintenance { + internal static void Defragment(Stream archive, DefragOptions options) { + ArgumentNullException.ThrowIfNull(options); + if (options.Mode != DefragMode.ConsolidateAtStart) + throw new NotSupportedException($"UEFI FV defrag supports only {DefragMode.ConsolidateAtStart}."); + + var (image, volume) = OpenWritable(archive); + options.CancellationToken.ThrowIfCancellationRequested(); + options.OnProgress?.Invoke(new DefragProgressEvent("scanning", 0, -1, -1, image.Length, null, "Scanning UEFI firmware volume")); + + var output = image.ToArray(); + output.AsSpan(volume.DataStart, volume.End - volume.DataStart).Fill(volume.EraseByte); + var live = UefiFvParser.LiveSlots(volume).ToList(); + var anchored = live.Where(IsAnchored).ToList(); + foreach (var slot in anchored) + image.AsSpan(slot.Offset, slot.Size).CopyTo(output.AsSpan(slot.Offset, slot.Size)); + + var movable = live.Where(slot => !IsAnchored(slot)).ToList(); + for (var i = 0; i < movable.Count; i++) { + options.CancellationToken.ThrowIfCancellationRequested(); + var slot = movable[i]; + var footprint = UefiFvConstants.Align8(slot.Size); + var destination = FindRun(output, volume, footprint); + if (destination < 0) + throw new InvalidDataException($"UEFI FV cannot place {slot.Name:D} while preserving fixed/aligned files."); + image.AsSpan(slot.Offset, slot.Size).CopyTo(output.AsSpan(destination, slot.Size)); + options.OnProgress?.Invoke(new DefragProgressEvent("writing", (i + 1d) / Math.Max(1, movable.Count), + slot.Offset, destination, image.Length, null, UefiFvWriter.EntryName(slot.Name, slot.Type))); + } + + var packed = UefiFvParser.Parse(output, volume.Start); + var usedEnd = UefiFvParser.LiveSlots(packed).Select(slot => slot.DataEnd).DefaultIfEmpty(packed.DataStart).Max(); + UefiFvParser.WriteUsedSize(output, packed, usedEnd); + VerifyPayloads(image, output, volume.Start); + options.CancellationToken.ThrowIfCancellationRequested(); + Commit(archive, output); + options.OnProgress?.Invoke(new DefragProgressEvent("complete", 1, -1, -1, output.Length, null, "UEFI FV defragmented")); + } + + internal static void Purge(Stream archive) { + var (image, volume) = OpenWritable(archive); + image.AsSpan(volume.DataStart, volume.End - volume.DataStart).Fill(volume.EraseByte); + UefiFvParser.WriteUsedSize(image, volume, volume.DataStart); + Commit(archive, image); + } + + internal static (byte[] Image, UefiFvLayout Volume) Read(Stream archive, bool writable) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanSeek || writable && !archive.CanWrite) + throw new ArgumentException(writable ? "UEFI FV maintenance requires a readable/writable/seekable stream." : "UEFI FV layout requires a readable/seekable stream.", nameof(archive)); + if (archive.Length > int.MaxValue) + throw new NotSupportedException("The in-memory UEFI FV implementation supports images up to 2 GiB."); + archive.Position = 0; + var image = new byte[checked((int)archive.Length)]; + archive.ReadExactly(image); + var start = UefiFvReader.FindFirst(image) ?? throw new InvalidDataException("UEFI FV header was not found."); + return (image, UefiFvParser.Parse(image, start)); + } + + internal static void EnsureMutable(UefiFvLayout volume) { + if (volume.FileSystemGuid != UefiFvConstants.Ffs2Guid && volume.FileSystemGuid != UefiFvConstants.Ffs3Guid) + throw new NotSupportedException($"UEFI FV uses unsupported file-system GUID {volume.FileSystemGuid:D}."); + if (volume.IsSigned) + throw new NotSupportedException("Signed UEFI firmware volumes cannot be modified without re-signing the complete volume."); + } + + internal static void Commit(Stream archive, byte[] image) { + archive.Position = 0; + archive.Write(image); + archive.SetLength(image.Length); + } + + private static (byte[] Image, UefiFvLayout Volume) OpenWritable(Stream archive) { + var result = Read(archive, writable: true); + EnsureMutable(result.Volume); + return result; + } + + private static bool IsAnchored(UefiFvSlot slot) + => slot.IsPad || slot.IsFixed || slot.Name == UefiFvConstants.VolumeTopGuid || slot.DataAlignment > 1; + + private static int FindRun(byte[] image, UefiFvLayout volume, int footprint) { + for (var p = volume.DataStart; p + footprint <= volume.End; p += UefiFvWriter.Alignment) + if (UefiFvParser.IsErased(image.AsSpan(p, footprint), volume.EraseByte)) return p; + return -1; + } + + private static void VerifyPayloads(byte[] before, byte[] after, int start) { + var expected = UefiFvReader.Read(before, start).Files.Where(f => f.Type != 0xF0).ToList(); + var actual = UefiFvReader.Read(after, start).Files.Where(f => f.Type != 0xF0).ToList(); + if (expected.Count != actual.Count) throw new InvalidDataException("UEFI FV defrag changed the live-file count."); + foreach (var file in expected) { + var index = actual.FindIndex(candidate => candidate.Name == file.Name && candidate.Type == file.Type + && candidate.Contents.AsSpan().SequenceEqual(file.Contents)); + if (index < 0) throw new InvalidDataException($"UEFI FV defrag changed payload {file.Name:D}."); + actual.RemoveAt(index); + } + } +} diff --git a/FileFormats/FileFormat.UefiFv/UefiFvParser.cs b/FileFormats/FileFormat.UefiFv/UefiFvParser.cs new file mode 100644 index 000000000..bbf4f99f3 --- /dev/null +++ b/FileFormats/FileFormat.UefiFv/UefiFvParser.cs @@ -0,0 +1,105 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; + +namespace FileFormat.UefiFv; + +internal static class UefiFvParser { + internal static UefiFvLayout Parse(ReadOnlySpan data, int start) { + if (start < 0 || start > data.Length - 56) + throw new InvalidDataException("UefiFv: file shorter than minimum FV header."); + if (!data.Slice(start + UefiFvReader.SignatureOffset, 4).SequenceEqual(UefiFvReader.Signature)) + throw new InvalidDataException($"UefiFv: '_FVH' signature not found at offset {start + UefiFvReader.SignatureOffset}."); + + var fsGuid = new Guid(data.Slice(start + 16, 16)); + var length64 = BinaryPrimitives.ReadUInt64LittleEndian(data[(start + 32)..]); + if (length64 < 56 || length64 > int.MaxValue) + throw new NotSupportedException($"UefiFv: unsupported firmware-volume length {length64} bytes."); + var end = checked(start + (int)length64); + if (end > data.Length) throw new InvalidDataException("UefiFv: firmware volume extends past the image."); + + var attributes = BinaryPrimitives.ReadUInt32LittleEndian(data[(start + 44)..]); + var erasePolarity = (attributes & UefiFvConstants.ErasePolarityMask) != 0; + var eraseByte = erasePolarity ? (byte)0xFF : (byte)0x00; + var headerLength = BinaryPrimitives.ReadUInt16LittleEndian(data[(start + 48)..]); + if (headerLength < 56 || start + headerLength > end) throw new InvalidDataException("UefiFv: invalid header length."); + var dataStart = start + headerLength; + int? usedSizeOffset = null; + var signed = false; + + var extOffset = BinaryPrimitives.ReadUInt16LittleEndian(data[(start + 52)..]); + if (extOffset != 0) { + var ext = checked(start + extOffset); + if (ext < start || ext > end - 20) throw new InvalidDataException("UefiFv: invalid extended-header offset."); + var extSizeRaw = BinaryPrimitives.ReadUInt32LittleEndian(data[(ext + 16)..]); + if (extSizeRaw < 20 || extSizeRaw > int.MaxValue) throw new InvalidDataException("UefiFv: invalid extended-header size."); + var extEnd = checked(ext + (int)extSizeRaw); + if (extEnd > end) throw new InvalidDataException("UefiFv: extended header extends past the volume."); + dataStart = Math.Max(dataStart, extEnd); + for (var p = ext + 20; p + 4 <= extEnd;) { + var entrySize = BinaryPrimitives.ReadUInt16LittleEndian(data[p..]); + var entryType = BinaryPrimitives.ReadUInt16LittleEndian(data[(p + 2)..]); + if (entrySize < 4 || p + entrySize > extEnd) throw new InvalidDataException("UefiFv: invalid extension entry."); + if (entryType == 3 && entrySize >= 8) usedSizeOffset = p + 4; + if (entryType == 2 && entrySize >= 20 && new Guid(data.Slice(p + 4, 16)) == UefiFvConstants.SignedContentsGuid) + signed = true; + p += entrySize; + } + } + + dataStart = UefiFvConstants.AlignOffset(start, dataStart); + if (dataStart > end) throw new InvalidDataException("UefiFv: file area lies beyond the volume."); + var slots = ReadSlots(data, start, end, dataStart, erasePolarity, eraseByte); + return new UefiFvLayout(start, end, dataStart, fsGuid, attributes, erasePolarity, eraseByte, usedSizeOffset, signed, slots); + } + + internal static IEnumerable LiveSlots(UefiFvLayout volume, bool includePad = true) { + foreach (var slot in volume.Slots) + if ((includePad || !slot.IsPad) && IsLive(volume, slot)) yield return slot; + } + + internal static bool IsLive(UefiFvLayout volume, UefiFvSlot slot) { + if (slot.State == UefiFvConstants.DataValid) return true; + if (slot.State != UefiFvConstants.MarkedForUpdate) return false; + return !volume.Slots.Any(other => other.Offset > slot.Offset && other.Name == slot.Name && other.State == UefiFvConstants.DataValid); + } + + internal static bool IsDiscardable(UefiFvLayout volume, UefiFvSlot slot) + => slot.State is UefiFvConstants.Deleted or UefiFvConstants.HeaderInvalid + || slot.State == UefiFvConstants.MarkedForUpdate && !IsLive(volume, slot); + + internal static bool IsErased(ReadOnlySpan data, byte eraseByte) { + foreach (var value in data) if (value != eraseByte) return false; + return true; + } + + internal static void WriteUsedSize(Span image, UefiFvLayout volume, int usedEnd) { + if (volume.UsedSizeOffset is { } offset) + BinaryPrimitives.WriteUInt32LittleEndian(image[offset..], checked((uint)(usedEnd - volume.Start))); + } + + private static List ReadSlots(ReadOnlySpan data, int start, int end, int dataStart, + bool erasePolarity, byte eraseByte) { + var result = new List(); + for (var p = dataStart; p + 24 <= end;) { + var quantum = Math.Min(8, end - p); + if (IsErased(data.Slice(p, quantum), eraseByte)) { p += quantum; continue; } + var attr = data[p + 19]; + var large = (attr & UefiFvConstants.LargeFile) != 0; + var headerSize = large ? 32 : 24; + if (p + headerSize > end) throw new InvalidDataException("UefiFv: truncated FFS header."); + var size24 = (uint)(data[p + 20] | (data[p + 21] << 8) | (data[p + 22] << 16)); + var size64 = large ? BinaryPrimitives.ReadUInt64LittleEndian(data[(p + 24)..]) : size24; + if (large && size24 != 0) throw new InvalidDataException("UefiFv: large FFS header has non-zero Size[3]."); + if (size64 < (ulong)headerSize || size64 > int.MaxValue || p + (long)size64 > end) + throw new InvalidDataException($"UefiFv: invalid FFS size at FV offset 0x{p - start:X}."); + var size = (int)size64; + var aligned = UefiFvConstants.Align8(size); + var footprint = p + (long)aligned <= end ? aligned : size; + var rawState = data[p + 23]; + result.Add(new UefiFvSlot(p, headerSize, size, footprint, new Guid(data.Slice(p, 16)), data[p + 18], attr, + rawState, UefiFvConstants.EffectiveState(rawState, erasePolarity))); + p = checked(p + footprint); + } + return result; + } +} diff --git a/FileFormats/FileFormat.UefiFv/UefiFvReader.cs b/FileFormats/FileFormat.UefiFv/UefiFvReader.cs index 5e57d91bc..d0871be0e 100644 --- a/FileFormats/FileFormat.UefiFv/UefiFvReader.cs +++ b/FileFormats/FileFormat.UefiFv/UefiFvReader.cs @@ -1,15 +1,9 @@ #pragma warning disable CS1591 using System.Buffers.Binary; -using System.Text; namespace FileFormat.UefiFv; -/// -/// Reader for UEFI Platform Initialization (PI) Firmware Volumes. Locates the -/// FV header by scanning for the _FVH signature at offset 40 from the -/// start of each 16-byte-aligned candidate (UEFI PI Volume 3). Walks the FFS -/// file list and returns one record per live file. -/// +/// Reader for UEFI PI firmware volumes and standard FFS2/FFS3 file records. public sealed class UefiFvReader { public static readonly byte[] Signature = [(byte)'_', (byte)'F', (byte)'V', (byte)'H']; public const int SignatureOffset = 40; @@ -23,33 +17,16 @@ public sealed record FvHeader( ushort ExtHeaderOffset, byte Revision, IReadOnlyList<(uint NumBlocks, uint Length)> BlockMap - ); - - public sealed record FfsFile( - Guid Name, - byte Type, - byte Attributes, - byte State, - uint Size, - byte[] Contents - ); + ) { + public bool ErasePolarity => (this.Attributes & UefiFvConstants.ErasePolarityMask) != 0; + public byte EraseByte => this.ErasePolarity ? (byte)0xFF : (byte)0x00; + } - public sealed record FirmwareVolume( - int StartOffset, - FvHeader Header, - IReadOnlyList Files - ); + public sealed record FfsFile(Guid Name, byte Type, byte Attributes, byte State, uint Size, byte[] Contents); + public sealed record FirmwareVolume(int StartOffset, FvHeader Header, IReadOnlyList Files); public static FirmwareVolume Read(ReadOnlySpan data, int fvStart = 0) { - if (data.Length < fvStart + 56) - throw new InvalidDataException("UefiFv: file shorter than minimum FV header."); - - var sigSpan = data.Slice(fvStart + SignatureOffset, 4); - if (!sigSpan.SequenceEqual(Signature)) - throw new InvalidDataException( - $"UefiFv: '_FVH' signature not found at offset {fvStart + SignatureOffset}."); - - var fsGuid = new Guid(data.Slice(fvStart + 16, 16)); + var layout = UefiFvParser.Parse(data, fvStart); var fvLength = BinaryPrimitives.ReadUInt64LittleEndian(data[(fvStart + 32)..]); var attributes = BinaryPrimitives.ReadUInt32LittleEndian(data[(fvStart + 44)..]); var headerLength = BinaryPrimitives.ReadUInt16LittleEndian(data[(fvStart + 48)..]); @@ -58,71 +35,34 @@ public static FirmwareVolume Read(ReadOnlySpan data, int fvStart = 0) { var revision = data[fvStart + 55]; var blockMap = new List<(uint, uint)>(); - var p = fvStart + 56; - while (p + 8 <= data.Length) { - var nb = BinaryPrimitives.ReadUInt32LittleEndian(data[p..]); - var bl = BinaryPrimitives.ReadUInt32LittleEndian(data[(p + 4)..]); - p += 8; - if (nb == 0 && bl == 0) break; - blockMap.Add((nb, bl)); + var headerEnd = checked(fvStart + headerLength); + var terminated = false; + for (var p = fvStart + 56; p + 8 <= headerEnd; p += 8) { + var blocks = BinaryPrimitives.ReadUInt32LittleEndian(data[p..]); + var length = BinaryPrimitives.ReadUInt32LittleEndian(data[(p + 4)..]); + if (blocks == 0 && length == 0) { terminated = true; break; } + blockMap.Add((blocks, length)); } + if (!terminated) throw new InvalidDataException("UefiFv: firmware-volume block map is not terminated inside HeaderLength."); - var header = new FvHeader(fsGuid, fvLength, attributes, headerLength, checksum, extOff, revision, blockMap); - var ffsStart = fvStart + headerLength; - var ffsEnd = checked((int)Math.Min((long)data.Length, fvStart + (long)fvLength)); - var files = ReadFfsFiles(data, ffsStart, ffsEnd); + // `data` is a ReadOnlySpan, which a lambda cannot capture, so the slices are materialised + // here rather than inside the projection. + var files = new List(); + foreach (var slot in UefiFvParser.LiveSlots(layout)) + files.Add(new FfsFile( + slot.Name, slot.Type, slot.Attributes, slot.RawState, checked((uint)slot.Size), + data.Slice(slot.DataOffset, slot.DataLength).ToArray())); + var header = new FvHeader(new Guid(data.Slice(fvStart + 16, 16)), fvLength, attributes, + headerLength, checksum, extOff, revision, blockMap); return new FirmwareVolume(fvStart, header, files); } public static int? FindFirst(ReadOnlySpan data) { - for (var i = 0; i + SignatureOffset + 4 <= data.Length; i += 16) { - if (data.Slice(i + SignatureOffset, 4).SequenceEqual(Signature)) - return i; - } + for (var i = 0; i + SignatureOffset + 4 <= data.Length; i += 16) + if (data.Slice(i + SignatureOffset, 4).SequenceEqual(Signature)) return i; return null; } - private static List ReadFfsFiles(ReadOnlySpan data, int start, int end) { - var files = new List(); - var pos = Align8(start); - while (pos + 24 <= end) { - // Free/deleted regions may occur between live files after offline - // mutation, so a gap has to be walked over one alignment quantum at a - // time. Only the quantum itself is tested: asking whether a whole 24-byte - // header is erased reads past the end of the gap, and the last two - // quanta of a gap then look half-written — the walk parsed that overlap - // as a file, took its size from the middle of the real header's GUID, and - // gave up on the volume. An FFS file starts on this boundary and opens - // with its GUID, which is never all-ones. - if (IsErased(data.Slice(pos, 8))) { - pos += 8; - continue; - } - - var header = data.Slice(pos, 24); - - var name = new Guid(header[..16]); - var type = header[18]; - var attrs = header[19]; - var size = (uint)(header[20] | (header[21] << 8) | (header[22] << 16)); - var state = header[23]; - if (size < 24 || pos + (long)size > end) break; - - var contents = data.Slice(pos + 24, checked((int)size - 24)).ToArray(); - files.Add(new FfsFile(name, type, attrs, state, size, contents)); - pos = Align8(pos + checked((int)size)); - } - return files; - - static bool IsErased(ReadOnlySpan bytes) { - foreach (var b in bytes) - if (b != 0xFF) return false; - return true; - } - - static int Align8(int v) => (v + 7) & ~7; - } - public static string FileTypeName(byte t) => t switch { 0x00 => "EFI_FV_FILETYPE_ALL", 0x01 => "EFI_FV_FILETYPE_RAW", @@ -145,8 +85,8 @@ static bool IsErased(ReadOnlySpan bytes) { }; public static string ShortTypeTag(byte t) { - var n = FileTypeName(t); + var name = FileTypeName(t); const string prefix = "EFI_FV_FILETYPE_"; - return n.StartsWith(prefix, StringComparison.Ordinal) ? n[prefix.Length..] : n; + return name.StartsWith(prefix, StringComparison.Ordinal) ? name[prefix.Length..] : name; } } diff --git a/FileFormats/FileFormat.UefiFv/UefiFvWriter.cs b/FileFormats/FileFormat.UefiFv/UefiFvWriter.cs index cc6a148da..90c2cd305 100644 --- a/FileFormats/FileFormat.UefiFv/UefiFvWriter.cs +++ b/FileFormats/FileFormat.UefiFv/UefiFvWriter.cs @@ -5,28 +5,32 @@ namespace FileFormat.UefiFv; -/// Writes a standalone PI firmware volume containing ordinary FFS2 files. +/// Writes standalone PI firmware volumes containing ordinary FFS2/FFS3 files. internal static class UefiFvWriter { internal const int HeaderLength = 72; internal const int FfsHeaderLength = 24; + internal const int FfsHeader2Length = 32; internal const int Alignment = 8; internal const int DefaultReserve = 64 * 1024; internal const uint FvAttributes = 0x0004FEFF; - internal static readonly Guid Ffs2Guid = Guid.Parse("8C8CE578-8A3D-4F1C-9935-896185C32DD3"); + internal static readonly Guid Ffs2Guid = UefiFvConstants.Ffs2Guid; + internal static readonly Guid Ffs3Guid = UefiFvConstants.Ffs3Guid; internal readonly record struct FileIdentity(Guid Guid, byte Type); public static byte[] Build(IEnumerable<(string Name, byte[] Data)> inputs) { var files = inputs.Select(i => (Identity: IdentityFromName(i.Name), i.Data)).ToList(); - var used = HeaderLength + files.Sum(f => Align8(checked(FfsHeaderLength + f.Data.Length))); + var useFfs3 = files.Any(f => RequiresLargeFile(f.Data.Length)); + var fileSystemGuid = useFfs3 ? Ffs3Guid : Ffs2Guid; + var used = HeaderLength + files.Sum(f => Align8(checked(HeaderSize(f.Data.Length) + f.Data.Length))); var capacity = Align4K(checked(used + DefaultReserve)); var image = new byte[capacity]; image.AsSpan().Fill(0xFF); - WriteVolumeHeader(image); + WriteVolumeHeader(image, fileSystemGuid); var position = HeaderLength; foreach (var file in files) { - var encoded = BuildFfsFile(file.Identity.Guid, file.Identity.Type, file.Data); + var encoded = BuildFfsFile(file.Identity.Guid, file.Identity.Type, file.Data, fileSystemGuid, erasePolarity: true); encoded.CopyTo(image, position); position += Align8(encoded.Length); } @@ -42,32 +46,44 @@ internal static FileIdentity IdentityFromName(string name) { } var hash = SHA256.HashData(Encoding.UTF8.GetBytes(name.Replace('\\', '/'))); - return new FileIdentity(new Guid(hash.AsSpan(0, 16)), 0x01); // RAW + return new FileIdentity(new Guid(hash.AsSpan(0, 16)), 0x01); } - internal static byte[] BuildFfsFile(Guid guid, byte type, ReadOnlySpan contents) { - var size = checked(FfsHeaderLength + contents.Length); - if (size > 0xFFFFFF) - throw new NotSupportedException("FFS2 files larger than 16 MiB require EFI_FFS_FILE_HEADER2."); + internal static byte[] BuildFfsFile(Guid guid, byte type, ReadOnlySpan contents) + => BuildFfsFile(guid, type, contents, Ffs2Guid, erasePolarity: true); + internal static byte[] BuildFfsFile(Guid guid, byte type, ReadOnlySpan contents, + Guid fileSystemGuid, bool erasePolarity) { + var large = RequiresLargeFile(contents.Length); + if (large && fileSystemGuid != Ffs3Guid) + throw new NotSupportedException("FFS files larger than 16 MiB require an EFI_FIRMWARE_FILE_SYSTEM3_GUID volume."); + + var headerSize = large ? FfsHeader2Length : FfsHeaderLength; + var size = checked(headerSize + contents.Length); var file = new byte[size]; guid.TryWriteBytes(file.AsSpan(0, 16)); - file[17] = 0xAA; // fixed checksum when FFS_ATTRIB_CHECKSUM is clear + file[17] = 0xAA; file[18] = type; - file[19] = 0; - file[20] = (byte)size; - file[21] = (byte)(size >> 8); - file[22] = (byte)(size >> 16); - file[23] = 0xF8; - contents.CopyTo(file.AsSpan(FfsHeaderLength)); - file[16] = HeaderChecksum(file.AsSpan(0, FfsHeaderLength)); + file[19] = large ? UefiFvConstants.LargeFile : (byte)0; + if (large) + BinaryPrimitives.WriteUInt64LittleEndian(file.AsSpan(24), (ulong)size); + else { + file[20] = (byte)size; + file[21] = (byte)(size >> 8); + file[22] = (byte)(size >> 16); + } + file[23] = UefiFvConstants.EncodeState( + UefiFvConstants.HeaderConstruction | UefiFvConstants.HeaderValid | UefiFvConstants.DataValid, + erasePolarity); + contents.CopyTo(file.AsSpan(headerSize)); + file[16] = HeaderChecksum(file.AsSpan(0, headerSize)); return file; } - internal static void WriteVolumeHeader(Span image) { + internal static void WriteVolumeHeader(Span image, Guid? fileSystemGuid = null) { if (image.Length < HeaderLength) throw new ArgumentException("FV buffer is too small.", nameof(image)); image[..HeaderLength].Clear(); - Ffs2Guid.TryWriteBytes(image.Slice(16, 16)); + (fileSystemGuid ?? Ffs2Guid).TryWriteBytes(image.Slice(16, 16)); BinaryPrimitives.WriteUInt64LittleEndian(image[32..], (ulong)image.Length); "_FVH"u8.CopyTo(image[40..]); BinaryPrimitives.WriteUInt32LittleEndian(image[44..], FvAttributes); @@ -98,8 +114,10 @@ private static ushort VolumeHeaderChecksum(ReadOnlySpan header) { return unchecked((ushort)(0 - sum)); } - internal static int Align8(int value) => checked((value + Alignment - 1) & ~(Alignment - 1)); + internal static int Align8(int value) => UefiFvConstants.Align8(value); private static int Align4K(int value) => checked((value + 4095) & ~4095); + private static bool RequiresLargeFile(int dataLength) => (long)FfsHeaderLength + dataLength > 0xFFFFFF; + private static int HeaderSize(int dataLength) => RequiresLargeFile(dataLength) ? FfsHeader2Length : FfsHeaderLength; internal static string EntryName(Guid guid, byte type) => $"{guid:D}_{UefiFvReader.ShortTypeTag(type)}.bin"; @@ -107,22 +125,11 @@ internal static string EntryName(Guid guid, byte type) private static byte ParseTypeTag(string tail) { var tag = Path.GetFileNameWithoutExtension(tail).TrimStart('_').ToUpperInvariant(); return tag switch { - "RAW" => 0x01, - "FREEFORM" => 0x02, - "SECURITY_CORE" => 0x03, - "PEI_CORE" => 0x04, - "DXE_CORE" => 0x05, - "PEIM" => 0x06, - "DRIVER" => 0x07, - "COMBINED_PEIM_DRIVER" => 0x08, - "APPLICATION" => 0x09, - "MM" => 0x0A, - "FIRMWARE_VOLUME_IMAGE" => 0x0B, - "COMBINED_MM_DXE" => 0x0C, - "MM_CORE" => 0x0D, - "MM_STANDALONE" => 0x0E, - "MM_CORE_STANDALONE" => 0x0F, - _ => 0x01, + "RAW" => 0x01, "FREEFORM" => 0x02, "SECURITY_CORE" => 0x03, "PEI_CORE" => 0x04, + "DXE_CORE" => 0x05, "PEIM" => 0x06, "DRIVER" => 0x07, "COMBINED_PEIM_DRIVER" => 0x08, + "APPLICATION" => 0x09, "MM" => 0x0A, "FIRMWARE_VOLUME_IMAGE" => 0x0B, + "COMBINED_MM_DXE" => 0x0C, "MM_CORE" => 0x0D, "MM_STANDALONE" => 0x0E, + "MM_CORE_STANDALONE" => 0x0F, _ => 0x01, }; } } diff --git a/FileFormats/FileFormat.Zip/ZipLayoutMap.cs b/FileFormats/FileFormat.Zip/ZipLayoutMap.cs index a4866626e..bf55fd6ef 100644 --- a/FileFormats/FileFormat.Zip/ZipLayoutMap.cs +++ b/FileFormats/FileFormat.Zip/ZipLayoutMap.cs @@ -1,23 +1,34 @@ #pragma warning disable CS1591 +using System.Buffers.Binary; using Compression.Registry; namespace FileFormat.Zip; /// -/// Walks the ZIP central directory and emits the byte-level layout of every -/// local file header, compressed data payload, the central directory itself, -/// and the EOCD record as tiles. +/// Walks a ZIP central directory and emits a fail-closed byte-level layout of local headers, +/// compressed payloads, optional data descriptors, the central directory and EOCD. /// public static class ZipLayoutMap { - /// - /// Enumerates the value. - /// + private sealed record EntryLayout( + ZipEntry Entry, + long HeaderOffset, + int HeaderLength, + long DataOffset, + long DataLength, + int DescriptorLength) { + public long End => this.DataOffset + this.DataLength + this.DescriptorLength; + } + public static IEnumerable Enumerate(Stream archive) { ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanSeek) + yield break; + archive.Position = 0; - long cdOffset, cdSize; + long cdOffset; + long cdSize; int cdCount; long eocdOffset; try { @@ -27,53 +38,69 @@ public static IEnumerable Enumerate(Stream archive) { yield break; } - // Read central directory entries to learn local header offsets - archive.Position = cdOffset; - var reader = new BinaryReader(archive, System.Text.Encoding.Latin1, leaveOpen: true); - var entries = new List(); - for (var i = 0; i < cdCount; i++) { - try { - entries.Add(ZipCentralDirectoryEntry.Read(reader)); - } catch { - break; - } + List entries; + try { + entries = ReadCentralDirectory(archive, cdOffset, cdCount); + } catch { + yield break; } - // For each entry, emit local file header (MetadataReserved) + compressed data (Used) - foreach (var entry in entries) { - // Determine LFH size: fixed 30 bytes + filename + extra field - archive.Position = entry.LocalHeaderOffset; - if (archive.Position + 30 > archive.Length) + var ordered = entries.OrderBy(entry => entry.LocalHeaderOffset).ToArray(); + var layouts = new List(ordered.Length); + for (var i = 0; i < ordered.Length; ++i) { + var entry = ordered[i]; + if (!TryReadLocalHeaderLength(archive, entry.LocalHeaderOffset, out var headerLength)) continue; - var lfhBuf = new byte[30]; - if (archive.Read(lfhBuf, 0, 30) < 30) + var dataOffset = entry.LocalHeaderOffset + headerLength; + if (dataOffset < 0 || entry.CompressedSize < 0 || dataOffset > archive.Length - entry.CompressedSize) continue; - var fnLen = BitConverter.ToUInt16(lfhBuf, 26); - var exLen = BitConverter.ToUInt16(lfhBuf, 28); - var lfhSize = 30 + fnLen + exLen; + var dataEnd = dataOffset + entry.CompressedSize; + var nextBoundary = cdOffset; + for (var next = i + 1; next < ordered.Length; ++next) { + if (ordered[next].LocalHeaderOffset > entry.LocalHeaderOffset) { + nextBoundary = Math.Min(nextBoundary, ordered[next].LocalHeaderOffset); + break; + } + } + nextBoundary = Math.Clamp(nextBoundary, dataEnd, archive.Length); - // MetadataReserved tile for the local file header - yield return new DefragBlockInfo( + var descriptorLength = ReadDataDescriptorLength(archive, entry, dataEnd, nextBoundary); + layouts.Add(new EntryLayout( + entry, entry.LocalHeaderOffset, - lfhSize, + headerLength, + dataOffset, + entry.CompressedSize, + descriptorLength)); + } + + foreach (var layout in layouts) { + yield return new DefragBlockInfo( + layout.HeaderOffset, + layout.HeaderLength, DefragBlockKind.MetadataReserved, - FileName: $"LFH: {entry.FileName}"); + FileName: $"LFH: {layout.Entry.FileName}"); - // Used tile for compressed data - if (entry.CompressedSize > 0) { - var classification = ClassifyMethod(entry.CompressionMethod); + if (layout.DataLength > 0) { yield return new DefragBlockInfo( - entry.LocalHeaderOffset + lfhSize, - entry.CompressedSize, + layout.DataOffset, + layout.DataLength, DefragBlockKind.Used, - FileName: entry.FileName, - Classification: classification); + FileName: layout.Entry.FileName, + Classification: ClassifyMethod(layout.Entry.CompressionMethod)); + } + + if (layout.DescriptorLength > 0) { + yield return new DefragBlockInfo( + layout.DataOffset + layout.DataLength, + layout.DescriptorLength, + DefragBlockKind.MetadataReserved, + FileName: $"Data descriptor: {layout.Entry.FileName}"); } } - // Central directory = one MetadataReserved tile if (cdSize > 0) { yield return new DefragBlockInfo( cdOffset, @@ -82,33 +109,21 @@ public static IEnumerable Enumerate(Stream archive) { FileName: "Central Directory"); } - // EOCD = one MetadataReserved tile (from EOCD start to end of file) if (eocdOffset >= 0) { - var eocdSize = archive.Length - eocdOffset; yield return new DefragBlockInfo( eocdOffset, - eocdSize, + archive.Length - eocdOffset, DefragBlockKind.MetadataReserved, FileName: "End of Central Directory"); } - // Detect gaps (free/dead bytes) between entries - var regions = new List<(long Start, long End)>(); - foreach (var entry in entries) { - archive.Position = entry.LocalHeaderOffset; - if (archive.Position + 30 > archive.Length) continue; - var buf = new byte[30]; - if (archive.Read(buf, 0, 30) < 30) continue; - var fn = BitConverter.ToUInt16(buf, 26); - var ex = BitConverter.ToUInt16(buf, 28); - var totalEntrySize = 30 + fn + ex + entry.CompressedSize; - regions.Add((entry.LocalHeaderOffset, entry.LocalHeaderOffset + totalEntrySize)); - } - regions.Add((cdOffset, cdOffset + cdSize)); + var regions = new List<(long Start, long End)>(layouts.Count + 2); + regions.AddRange(layouts.Select(layout => (layout.HeaderOffset, layout.End))); + if (cdSize > 0) + regions.Add((cdOffset, cdOffset + cdSize)); if (eocdOffset >= 0) regions.Add((eocdOffset, archive.Length)); - - regions.Sort((a, b) => a.Start.CompareTo(b.Start)); + regions.Sort(static (left, right) => left.Start.CompareTo(right.Start)); var cursor = 0L; foreach (var (start, end) in regions) { @@ -119,35 +134,138 @@ public static IEnumerable Enumerate(Stream archive) { DefragBlockKind.Free, FileName: "Dead space"); } - if (end > cursor) cursor = end; + cursor = Math.Max(cursor, end); + } + + if (cursor < archive.Length) { + yield return new DefragBlockInfo( + cursor, + archive.Length - cursor, + DefragBlockKind.Free, + FileName: "Dead space"); } } + private static List ReadCentralDirectory(Stream archive, long cdOffset, int cdCount) { + archive.Position = cdOffset; + using var reader = new BinaryReader(archive, System.Text.Encoding.Latin1, leaveOpen: true); + var entries = new List(cdCount); + for (var i = 0; i < cdCount; ++i) + entries.Add(ZipCentralDirectoryEntry.Read(reader)); + return entries; + } + + private static bool TryReadLocalHeaderLength(Stream archive, long offset, out int length) { + length = 0; + if (offset < 0 || offset > archive.Length - 30) + return false; + + Span header = stackalloc byte[30]; + archive.Position = offset; + if (!ReadExactly(archive, header)) + return false; + if (BinaryPrimitives.ReadUInt32LittleEndian(header) != ZipConstants.LocalFileHeaderSignature) + return false; + + var fileNameLength = BinaryPrimitives.ReadUInt16LittleEndian(header[26..]); + var extraLength = BinaryPrimitives.ReadUInt16LittleEndian(header[28..]); + length = 30 + fileNameLength + extraLength; + return offset <= archive.Length - length; + } + + /// + /// Returns the number of bytes occupied by an optional ZIP data descriptor. When bit 3 says a + /// descriptor exists but its values cannot be proven, the maximum standard descriptor span up + /// to the next known structure is reserved rather than exposed as free space. + /// + private static int ReadDataDescriptorLength(Stream archive, ZipEntry entry, long offset, long nextBoundary) { + if ((entry.GeneralPurposeFlags & ZipConstants.FlagDataDescriptor) == 0) + return 0; + + var availableLong = Math.Min(24L, Math.Max(0, nextBoundary - offset)); + var available = (int)availableLong; + if (available == 0) + return 0; + + Span data = stackalloc byte[24]; + archive.Position = offset; + var read = ReadUpTo(archive, data[..available]); + if (read == 0) + return 0; + data = data[..read]; + + var signaturePresent = data.Length >= 4 && + BinaryPrimitives.ReadUInt32LittleEndian(data) == ZipConstants.DataDescriptorSignature; + + if (signaturePresent) { + if (Matches32BitDescriptor(data, 4, entry)) + return 16; + if (Matches64BitDescriptor(data, 4, entry)) + return 24; + } + + if (Matches32BitDescriptor(data, 0, entry)) + return 12; + if (Matches64BitDescriptor(data, 0, entry)) + return 20; + + return read; + } + + private static bool Matches32BitDescriptor(ReadOnlySpan data, int offset, ZipEntry entry) { + if (entry.CompressedSize > uint.MaxValue || entry.UncompressedSize > uint.MaxValue || data.Length < offset + 12) + return false; + return BinaryPrimitives.ReadUInt32LittleEndian(data[offset..]) == entry.Crc32 && + BinaryPrimitives.ReadUInt32LittleEndian(data[(offset + 4)..]) == (uint)entry.CompressedSize && + BinaryPrimitives.ReadUInt32LittleEndian(data[(offset + 8)..]) == (uint)entry.UncompressedSize; + } + + private static bool Matches64BitDescriptor(ReadOnlySpan data, int offset, ZipEntry entry) { + if (data.Length < offset + 20) + return false; + return BinaryPrimitives.ReadUInt32LittleEndian(data[offset..]) == entry.Crc32 && + BinaryPrimitives.ReadUInt64LittleEndian(data[(offset + 4)..]) == (ulong)entry.CompressedSize && + BinaryPrimitives.ReadUInt64LittleEndian(data[(offset + 12)..]) == (ulong)entry.UncompressedSize; + } + + private static bool ReadExactly(Stream stream, Span buffer) { + var total = 0; + while (total < buffer.Length) { + var read = stream.Read(buffer[total..]); + if (read <= 0) + return false; + total += read; + } + return true; + } + + private static int ReadUpTo(Stream stream, Span buffer) { + var total = 0; + while (total < buffer.Length) { + var read = stream.Read(buffer[total..]); + if (read <= 0) + break; + total += read; + } + return total; + } + private static DefragBlockClass ClassifyMethod(ZipCompressionMethod method) => method switch { ZipCompressionMethod.Store => DefragBlockClass.Frozen, - ZipCompressionMethod.Deflate => DefragBlockClass.Normal, - ZipCompressionMethod.Deflate64 => DefragBlockClass.Normal, + ZipCompressionMethod.Deflate or ZipCompressionMethod.Deflate64 => DefragBlockClass.Normal, ZipCompressionMethod.BZip2 => DefragBlockClass.Cold, - ZipCompressionMethod.Lzma => DefragBlockClass.Hot, - ZipCompressionMethod.Zstd => DefragBlockClass.Hot, - ZipCompressionMethod.Ppmd => DefragBlockClass.Hot, + ZipCompressionMethod.Lzma or ZipCompressionMethod.Zstd or ZipCompressionMethod.Ppmd => DefragBlockClass.Hot, _ => DefragBlockClass.Normal, }; private static long FindEocdOffset(Stream stream) { - var searchLen = Math.Min(stream.Length, 65557); - var searchStart = stream.Length - searchLen; - var buffer = new byte[searchLen]; + var searchLength = Math.Min(stream.Length, 65_557); + var searchStart = stream.Length - searchLength; + var buffer = new byte[checked((int)searchLength)]; stream.Position = searchStart; - var bytesRead = 0; - while (bytesRead < buffer.Length) { - var read = stream.Read(buffer, bytesRead, buffer.Length - bytesRead); - if (read == 0) break; - bytesRead += read; - } + var bytesRead = ReadUpTo(stream, buffer); for (var i = bytesRead - 22; i >= 0; --i) { - if (buffer[i] == 0x50 && buffer[i + 1] == 0x4B && - buffer[i + 2] == 0x05 && buffer[i + 3] == 0x06) + if (BinaryPrimitives.ReadUInt32LittleEndian(buffer.AsSpan(i, 4)) == ZipConstants.EndOfCentralDirectorySignature) return searchStart + i; } return -1; diff --git a/FileSystems/FileSystem.Ecryptfs/EcryptfsCodec.cs b/FileSystems/FileSystem.Ecryptfs/EcryptfsCodec.cs new file mode 100644 index 000000000..d7d0150db --- /dev/null +++ b/FileSystems/FileSystem.Ecryptfs/EcryptfsCodec.cs @@ -0,0 +1,555 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace FileSystem.Ecryptfs; + +internal static class EcryptfsCodec { + internal const uint MarkerMagic = 0x3C81B7F5; + internal const int FixedHeaderSize = 26; + internal const int DefaultExtentSize = 4096; + internal const int MinimumMetadataSize = 8192; + internal const byte CurrentFileVersion = 3; + internal const uint EncryptedFlag = 0x00000002; + internal const uint MetadataInXattrFlag = 0x00000004; + internal const int PassphraseHashIterations = 65536; + + private const byte Tag3 = 0x8C; + private const byte Tag11 = 0xED; + private const byte Tag1 = 0x01; + private const byte Aes128 = 0x07; + private const byte Aes192 = 0x08; + private const byte Aes256 = 0x09; + + private static ReadOnlySpan DefaultSalt => [0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77]; + private static ReadOnlySpan ConsoleName => "_CONSOLE"u8; + + internal sealed record PassphrasePacket( + byte CipherCode, + byte[] Salt, + byte[] EncryptedFileKey, + byte[] Signature + ) { + public int KeySize => CipherCode switch { + Aes128 => 16, + Aes192 => 24, + Aes256 => 32, + _ => 0, + }; + + public string CipherDescription => CipherCode switch { + Aes128 => "AES-128-CBC", + Aes192 => "AES-192-CBC", + Aes256 => "AES-256-CBC", + _ => $"cipher-0x{CipherCode:X2}", + }; + } + + internal sealed record Header( + ulong DecryptedSize, + uint MarkerWord, + uint Flags, + uint ExtentSize, + ushort HeaderExtentCount, + int MetadataSize, + int PacketEndOffset, + bool PaddingIsKnown, + IReadOnlyList PassphrasePackets + ) { + public byte FileVersion => (byte)(Flags >> 24); + + public long CanonicalLength { + get { + var extents = DecryptedSize == 0 ? 0UL : checked((DecryptedSize + ExtentSize - 1) / ExtentSize); + var dataBytes = checked(extents * ExtentSize); + var total = checked((ulong)MetadataSize + dataBytes); + return total <= long.MaxValue + ? (long)total + : throw new InvalidDataException("eCryptfs lower-file length exceeds the supported stream range."); + } + } + + public string CipherDescription + => PassphrasePackets.Count == 0 ? "encrypted" : PassphrasePackets[0].CipherDescription; + } + + internal static Header ReadHeader(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead || !stream.CanSeek) + throw new ArgumentException("eCryptfs parsing requires a readable, seekable stream.", nameof(stream)); + + var originalPosition = stream.Position; + try { + Span fixedHeader = stackalloc byte[FixedHeaderSize]; + stream.Position = 0; + ReadExactly(stream, fixedHeader); + + var extentSize = BinaryPrimitives.ReadUInt32BigEndian(fixedHeader[20..24]); + var headerExtentCount = BinaryPrimitives.ReadUInt16BigEndian(fixedHeader[24..26]); + ValidateGeometry(extentSize, headerExtentCount); + + var metadataSize64 = checked((ulong)extentSize * headerExtentCount); + if (metadataSize64 > int.MaxValue) + throw new InvalidDataException("eCryptfs metadata region is too large to parse."); + var metadataSize = (int)metadataSize64; + if (stream.Length < metadataSize) + throw new InvalidDataException("Truncated eCryptfs metadata region."); + + var metadata = new byte[metadataSize]; + stream.Position = 0; + ReadExactly(stream, metadata); + var header = ParseHeader(metadata); + if (stream.Length < header.CanonicalLength) + throw new InvalidDataException("Truncated eCryptfs ciphertext extents."); + return header; + } finally { + stream.Position = originalPosition; + } + } + + internal static void DecryptTo(Stream input, Stream output, Header header, string password) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(password); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("eCryptfs decryption requires a readable, seekable input.", nameof(input)); + if (!output.CanWrite) + throw new ArgumentException("eCryptfs decryption requires a writable output.", nameof(output)); + + var packet = FindMatchingPassphrasePacket(header, password, out var fekek); + try { + if (packet.KeySize == 0) + throw new NotSupportedException($"eCryptfs cipher code 0x{packet.CipherCode:X2} is not supported; AES-128/192/256 are supported."); + + var fileKey = DecryptFileKey(packet, fekek); + try { + var rootIv = MD5.HashData(fileKey); + var encrypted = new byte[header.ExtentSize]; + var remaining = header.DecryptedSize; + var extentIndex = 0L; + input.Position = header.MetadataSize; + + using var aes = Aes.Create(); + aes.Mode = CipherMode.CBC; + aes.Padding = PaddingMode.None; + aes.Key = fileKey; + + while (remaining > 0) { + ReadExactly(input, encrypted); + var iv = DeriveExtentIv(rootIv, extentIndex++); + using var transform = aes.CreateDecryptor(aes.Key, iv); + var plaintext = transform.TransformFinalBlock(encrypted, 0, encrypted.Length); + var toWrite = (int)Math.Min((ulong)plaintext.Length, remaining); + output.Write(plaintext, 0, toWrite); + remaining -= (ulong)toWrite; + CryptographicOperations.ZeroMemory(plaintext); + } + } finally { + CryptographicOperations.ZeroMemory(fileKey); + } + } finally { + CryptographicOperations.ZeroMemory(fekek); + } + } + + internal static void Create(Stream output, ReadOnlySpan plaintext, string password, int keySize = 16) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(password); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("eCryptfs creation requires a writable, seekable output.", nameof(output)); + if (keySize is not (16 or 24 or 32)) + throw new ArgumentOutOfRangeException(nameof(keySize), "AES key size must be 16, 24, or 32 bytes."); + + var passwordBytes = Encoding.UTF8.GetBytes(password); + if (passwordBytes.Length > 64) + throw new ArgumentException("eCryptfs passphrases are limited to 64 encoded bytes.", nameof(password)); + + var metadata = new byte[MinimumMetadataSize]; + BinaryPrimitives.WriteUInt64BigEndian(metadata.AsSpan(0, 8), (ulong)plaintext.Length); + + Span markerBytes = stackalloc byte[4]; + RandomNumberGenerator.Fill(markerBytes); + var markerWord = BinaryPrimitives.ReadUInt32BigEndian(markerBytes); + BinaryPrimitives.WriteUInt32BigEndian(metadata.AsSpan(8, 4), markerWord); + BinaryPrimitives.WriteUInt32BigEndian(metadata.AsSpan(12, 4), markerWord ^ MarkerMagic); + BinaryPrimitives.WriteUInt32BigEndian(metadata.AsSpan(16, 4), ((uint)CurrentFileVersion << 24) | EncryptedFlag); + BinaryPrimitives.WriteUInt32BigEndian(metadata.AsSpan(20, 4), DefaultExtentSize); + BinaryPrimitives.WriteUInt16BigEndian(metadata.AsSpan(24, 2), MinimumMetadataSize / DefaultExtentSize); + + var fileKey = RandomNumberGenerator.GetBytes(keySize); + var (fekek, signature) = DerivePassphraseMaterial(passwordBytes, DefaultSalt); + try { + var encryptedFileKey = EncryptFileKey(fileKey, fekek, keySize); + try { + var offset = FixedHeaderSize; + offset = WriteTag3(metadata, offset, keySize, encryptedFileKey, DefaultSalt); + offset = WriteTag11(metadata, offset, signature); + + output.Position = 0; + output.SetLength(0); + output.Write(metadata); + + if (!plaintext.IsEmpty) { + var rootIv = MD5.HashData(fileKey); + using var aes = Aes.Create(); + aes.Mode = CipherMode.CBC; + aes.Padding = PaddingMode.None; + aes.Key = fileKey; + + var plainExtent = new byte[DefaultExtentSize]; + var consumed = 0; + var extentIndex = 0L; + while (consumed < plaintext.Length) { + Array.Clear(plainExtent); + var count = Math.Min(DefaultExtentSize, plaintext.Length - consumed); + plaintext.Slice(consumed, count).CopyTo(plainExtent); + var iv = DeriveExtentIv(rootIv, extentIndex++); + using var transform = aes.CreateEncryptor(aes.Key, iv); + var encrypted = transform.TransformFinalBlock(plainExtent, 0, plainExtent.Length); + output.Write(encrypted); + CryptographicOperations.ZeroMemory(encrypted); + consumed += count; + } + CryptographicOperations.ZeroMemory(plainExtent); + } + output.Position = 0; + } finally { + CryptographicOperations.ZeroMemory(encryptedFileKey); + } + } finally { + CryptographicOperations.ZeroMemory(fileKey); + CryptographicOperations.ZeroMemory(fekek); + CryptographicOperations.ZeroMemory(signature); + CryptographicOperations.ZeroMemory(passwordBytes); + } + } + + internal static int ResolveKeySize(string? encryptionMethod) { + if (string.IsNullOrWhiteSpace(encryptionMethod)) return 16; + return encryptionMethod.Trim().ToLowerInvariant() switch { + "aes" or "aes128" or "aes-128" or "aes-128-cbc" => 16, + "aes192" or "aes-192" or "aes-192-cbc" => 24, + "aes256" or "aes-256" or "aes-256-cbc" => 32, + _ => throw new NotSupportedException($"Unsupported eCryptfs encryption method '{encryptionMethod}'. Use AES-128, AES-192, or AES-256."), + }; + } + + internal static long WipeUnusedSpace(Stream image) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("eCryptfs wipe requires a readable, writable, seekable stream.", nameof(image)); + + var header = ReadHeader(image); + long wiped = 0; + if (header.PaddingIsKnown && header.PacketEndOffset < header.MetadataSize) + wiped += ZeroRange(image, header.PacketEndOffset, header.MetadataSize - header.PacketEndOffset); + if (image.Length > header.CanonicalLength) + wiped += ZeroRange(image, header.CanonicalLength, image.Length - header.CanonicalLength); + image.Position = 0; + return wiped; + } + + internal static void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("eCryptfs shrink requires a readable, seekable input.", nameof(input)); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("eCryptfs shrink requires a writable, seekable output.", nameof(output)); + + var header = ReadHeader(input); + input.Position = 0; + output.Position = 0; + output.SetLength(0); + CopyExactly(input, output, header.CanonicalLength); + output.Position = 0; + } + + internal static void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("eCryptfs purge requires a readable, writable, seekable stream.", nameof(archive)); + + var header = ReadHeader(archive); + Span zeroSize = stackalloc byte[8]; + archive.Position = 0; + archive.Write(zeroSize); + archive.SetLength(header.MetadataSize); + archive.Position = 0; + } + + private static Header ParseHeader(ReadOnlySpan metadata) { + if (metadata.Length < FixedHeaderSize) + throw new InvalidDataException("Truncated eCryptfs fixed header."); + + var decryptedSize = BinaryPrimitives.ReadUInt64BigEndian(metadata[..8]); + var markerWord = BinaryPrimitives.ReadUInt32BigEndian(metadata[8..12]); + var markerCheck = BinaryPrimitives.ReadUInt32BigEndian(metadata[12..16]); + if ((markerWord ^ MarkerMagic) != markerCheck) + throw new InvalidDataException("Invalid eCryptfs marker pair."); + + var flags = BinaryPrimitives.ReadUInt32BigEndian(metadata[16..20]); + var version = (byte)(flags >> 24); + if (version is 0 or > CurrentFileVersion) + throw new InvalidDataException($"Unsupported eCryptfs file version {version}."); + if ((flags & EncryptedFlag) == 0) + throw new InvalidDataException("The eCryptfs header does not mark the lower file as encrypted."); + if ((flags & MetadataInXattrFlag) != 0) + throw new InvalidDataException("This standalone eCryptfs stream declares xattr metadata; the required xattr is not part of the stream."); + + var extentSize = BinaryPrimitives.ReadUInt32BigEndian(metadata[20..24]); + var headerExtentCount = BinaryPrimitives.ReadUInt16BigEndian(metadata[24..26]); + ValidateGeometry(extentSize, headerExtentCount); + var metadataSize64 = checked((ulong)extentSize * headerExtentCount); + if (metadataSize64 > (ulong)metadata.Length) + throw new InvalidDataException("Truncated eCryptfs metadata region."); + var metadataSize = (int)metadataSize64; + + var packets = new List(); + var offset = FixedHeaderSize; + var paddingKnown = true; + while (offset < metadataSize) { + var tag = metadata[offset]; + if (tag == 0) break; + + if (tag == Tag3) { + var tag3 = ParsePacket(metadata[..metadataSize], ref offset, Tag3); + var packet = ParseTag3(tag3); + if (offset >= metadataSize || metadata[offset] != Tag11) + throw new InvalidDataException("eCryptfs tag-3 packet is not followed by its tag-11 signature packet."); + var tag11 = ParsePacket(metadata[..metadataSize], ref offset, Tag11); + packet = packet with { Signature = ParseTag11Signature(tag11) }; + packets.Add(packet); + continue; + } + + if (tag == Tag1) { + _ = ParsePacket(metadata[..metadataSize], ref offset, Tag1); + continue; + } + + // Unknown non-zero metadata is potentially live. Stop parsing and refuse to + // call the remainder free space; this keeps wipe fail-closed. + paddingKnown = false; + offset = metadataSize; + break; + } + + return new Header(decryptedSize, markerWord, flags, extentSize, headerExtentCount, + metadataSize, offset, paddingKnown, packets); + } + + private static PassphrasePacket ParseTag3(ReadOnlySpan body) { + if (body.Length < 14) + throw new InvalidDataException("Truncated eCryptfs tag-3 packet."); + if (body[0] != 0x04) + throw new InvalidDataException($"Unsupported eCryptfs tag-3 version {body[0]}."); + if (body[2] != 0x03) + throw new InvalidDataException($"Unsupported eCryptfs S2K specifier {body[2]}."); + if (body[3] != 0x01) + throw new InvalidDataException($"Unsupported eCryptfs tag-3 hash identifier {body[3]}."); + if (body[12] != 0x60) + throw new InvalidDataException($"Unsupported eCryptfs S2K iteration code 0x{body[12]:X2}."); + + var encryptedKey = body[13..].ToArray(); + if (encryptedKey.Length == 0 || encryptedKey.Length > 64 || (encryptedKey.Length & 15) != 0) + throw new InvalidDataException("Invalid eCryptfs encrypted file-key length."); + return new PassphrasePacket(body[1], body[4..12].ToArray(), encryptedKey, []); + } + + private static byte[] ParseTag11Signature(ReadOnlySpan body) { + if (body.Length < 14 || body[0] != 0x62) + throw new InvalidDataException("Invalid eCryptfs tag-11 literal packet."); + var nameLength = body[1]; + var signatureOffset = 2 + nameLength + 4; + if (signatureOffset > body.Length || body.Length - signatureOffset != 8) + throw new InvalidDataException("Invalid eCryptfs tag-11 signature length."); + return body[signatureOffset..].ToArray(); + } + + private static ReadOnlySpan ParsePacket(ReadOnlySpan data, ref int offset, byte expectedTag) { + if (offset >= data.Length || data[offset++] != expectedTag) + throw new InvalidDataException($"Expected eCryptfs packet tag 0x{expectedTag:X2}."); + var bodyLength = ReadPacketLength(data, ref offset); + if (bodyLength < 0 || bodyLength > data.Length - offset) + throw new InvalidDataException("Truncated eCryptfs packet body."); + var body = data.Slice(offset, bodyLength); + offset += bodyLength; + return body; + } + + private static int ReadPacketLength(ReadOnlySpan data, ref int offset) { + if (offset >= data.Length) + throw new InvalidDataException("Truncated eCryptfs packet length."); + var first = data[offset++]; + if (first < 192) return first; + if (first < 224) { + if (offset >= data.Length) + throw new InvalidDataException("Truncated two-byte eCryptfs packet length."); + return ((first - 192) << 8) + data[offset++] + 192; + } + throw new InvalidDataException("Unsupported eCryptfs partial/five-byte packet length."); + } + + private static int WriteTag3(Span metadata, int offset, int keySize, ReadOnlySpan encryptedFileKey, ReadOnlySpan salt) { + var bodyLength = 13 + encryptedFileKey.Length; + metadata[offset++] = Tag3; + metadata[offset++] = checked((byte)bodyLength); + metadata[offset++] = 0x04; + metadata[offset++] = keySize switch { 16 => Aes128, 24 => Aes192, 32 => Aes256, _ => throw new ArgumentOutOfRangeException(nameof(keySize)) }; + metadata[offset++] = 0x03; + metadata[offset++] = 0x01; + salt.CopyTo(metadata[offset..]); + offset += salt.Length; + metadata[offset++] = 0x60; + encryptedFileKey.CopyTo(metadata[offset..]); + return offset + encryptedFileKey.Length; + } + + private static int WriteTag11(Span metadata, int offset, ReadOnlySpan signature) { + const int bodyLength = 22; + metadata[offset++] = Tag11; + metadata[offset++] = bodyLength; + metadata[offset++] = 0x62; + metadata[offset++] = 8; + ConsoleName.CopyTo(metadata[offset..]); + offset += ConsoleName.Length; + metadata.Slice(offset, 4).Clear(); + offset += 4; + signature.CopyTo(metadata[offset..]); + return offset + signature.Length; + } + + private static PassphrasePacket FindMatchingPassphrasePacket(Header header, string password, out byte[] fekek) { + var passwordBytes = Encoding.UTF8.GetBytes(password); + if (passwordBytes.Length > 64) + throw new CryptographicException("eCryptfs passphrase exceeds 64 encoded bytes."); + try { + foreach (var packet in header.PassphrasePackets) { + var material = DerivePassphraseMaterial(passwordBytes, packet.Salt); + if (CryptographicOperations.FixedTimeEquals(material.Signature, packet.Signature)) { + CryptographicOperations.ZeroMemory(material.Signature); + fekek = material.Fekek; + return packet; + } + CryptographicOperations.ZeroMemory(material.Fekek); + CryptographicOperations.ZeroMemory(material.Signature); + } + } finally { + CryptographicOperations.ZeroMemory(passwordBytes); + } + if (header.PassphrasePackets.Count == 0) + throw new NotSupportedException("This eCryptfs file has no passphrase tag-3 packet; private-key authentication is not supported."); + throw new CryptographicException("The supplied eCryptfs passphrase does not match any authentication-token signature."); + } + + private static (byte[] Fekek, byte[] Signature) DerivePassphraseMaterial(ReadOnlySpan password, ReadOnlySpan salt) { + var seed = new byte[salt.Length + password.Length]; + salt.CopyTo(seed); + password.CopyTo(seed.AsSpan(salt.Length)); + + var digest = SHA512.HashData(seed); + CryptographicOperations.ZeroMemory(seed); + for (var i = 1; i < PassphraseHashIterations; ++i) { + var next = SHA512.HashData(digest); + CryptographicOperations.ZeroMemory(digest); + digest = next; + } + + var sigHash = SHA512.HashData(digest); + var signature = sigHash.AsSpan(0, 8).ToArray(); + CryptographicOperations.ZeroMemory(sigHash); + return (digest, signature); + } + + private static byte[] EncryptFileKey(ReadOnlySpan fileKey, ReadOnlySpan fekek, int keySize) { + var padded = new byte[keySize == 24 ? 32 : keySize]; + fileKey.CopyTo(padded); + try { + using var aes = Aes.Create(); + aes.Mode = CipherMode.ECB; + aes.Padding = PaddingMode.None; + aes.Key = fekek[..keySize].ToArray(); + using var transform = aes.CreateEncryptor(); + return transform.TransformFinalBlock(padded, 0, padded.Length); + } finally { + CryptographicOperations.ZeroMemory(padded); + } + } + + private static byte[] DecryptFileKey(PassphrasePacket packet, ReadOnlySpan fekek) { + if (packet.KeySize == 0) + throw new NotSupportedException($"Unsupported eCryptfs cipher code 0x{packet.CipherCode:X2}."); + using var aes = Aes.Create(); + aes.Mode = CipherMode.ECB; + aes.Padding = PaddingMode.None; + aes.Key = fekek[..packet.KeySize].ToArray(); + using var transform = aes.CreateDecryptor(); + var decrypted = transform.TransformFinalBlock(packet.EncryptedFileKey, 0, packet.EncryptedFileKey.Length); + if (decrypted.Length < packet.KeySize) { + CryptographicOperations.ZeroMemory(decrypted); + throw new InvalidDataException("eCryptfs encrypted file key is shorter than the selected cipher key size."); + } + var fileKey = decrypted.AsSpan(0, packet.KeySize).ToArray(); + CryptographicOperations.ZeroMemory(decrypted); + return fileKey; + } + + private static byte[] DeriveExtentIv(ReadOnlySpan rootIv, long extentIndex) { + Span source = stackalloc byte[32]; + source.Clear(); + rootIv[..16].CopyTo(source); + var text = extentIndex.ToString(CultureInfo.InvariantCulture); + if (!Encoding.ASCII.TryGetBytes(text, source[16..31], out _)) + throw new InvalidDataException("eCryptfs extent index is too large for IV derivation."); + return MD5.HashData(source); + } + + private static void ValidateGeometry(uint extentSize, ushort headerExtentCount) { + if (extentSize < 16 || (extentSize & (extentSize - 1)) != 0 || (extentSize & 15) != 0) + throw new InvalidDataException($"Invalid eCryptfs extent size {extentSize}."); + if (headerExtentCount == 0) + throw new InvalidDataException("eCryptfs header extent count is zero."); + var metadataSize = checked((ulong)extentSize * headerExtentCount); + if (metadataSize < FixedHeaderSize) + throw new InvalidDataException("eCryptfs metadata region is smaller than the fixed header."); + } + + private static long ZeroRange(Stream stream, long offset, long count) { + if (count <= 0) return 0; + var zeroes = new byte[64 * 1024]; + stream.Position = offset; + var remaining = count; + while (remaining > 0) { + var chunk = (int)Math.Min(zeroes.Length, remaining); + stream.Write(zeroes, 0, chunk); + remaining -= chunk; + } + return count; + } + + private static void CopyExactly(Stream input, Stream output, long count) { + var buffer = new byte[64 * 1024]; + var remaining = count; + while (remaining > 0) { + var requested = (int)Math.Min(buffer.Length, remaining); + var read = input.Read(buffer, 0, requested); + if (read <= 0) throw new EndOfStreamException(); + output.Write(buffer, 0, read); + remaining -= read; + } + } + + private static void ReadExactly(Stream stream, Span buffer) { + var total = 0; + while (total < buffer.Length) { + var read = stream.Read(buffer[total..]); + if (read <= 0) throw new EndOfStreamException("Unexpected end of eCryptfs stream."); + total += read; + } + } + + private static void ReadExactly(Stream stream, byte[] buffer) => ReadExactly(stream, buffer.AsSpan()); +} diff --git a/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs b/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs index 1e12e1e77..a6fcb6bf0 100644 --- a/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs +++ b/FileSystems/FileSystem.Ecryptfs/EcryptfsFormatDescriptor.cs @@ -5,55 +5,194 @@ namespace FileSystem.Ecryptfs; /// -/// Read-only descriptor for eCryptfs per-file encryption containers. -/// eCryptfs (Linux) stacks on top of any underlying FS and stores each -/// encrypted file with a 4-byte big-endian marker 0x3C81B7F5 at -/// offset 0 followed by an 8-byte decrypted size, 4-byte flags, and -/// 4-byte extent-size hint. Decryption requires the user's passphrase + -/// EFEK packets — out of scope. The encrypted payload is surfaced as a -/// single opaque entry along with the parsed header metadata. +/// Descriptor for one eCryptfs lower file. eCryptfs is a stacked filesystem: +/// every lower file carries its own encrypted upper-file payload, size, extent +/// geometry and authentication-token packet set. /// -/// References: -/// -/// https://docs.kernel.org/filesystems/ecryptfs.html — Linux kernel eCryptfs documentation -/// https://github.com/torvalds/linux/tree/master/fs/ecryptfs — mainline implementation (ecryptfs_kernel.h defines the file-header marker + packet layout) -/// https://en.wikipedia.org/wiki/ECryptfs — Wikipedia overview -/// +/// +/// The implementation supports the ordinary passphrase path using the Linux +/// eCryptfs AES-128/192/256 packet conventions. Private-key authentication and +/// xattr-only metadata are deliberately rejected because their required key/xattr +/// material is not contained in a standalone stream. +/// /// -public sealed class EcryptfsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { +public sealed class EcryptfsFormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IArchiveCreatable, + IArchiveModifiable, + IArchiveShrinkable, + IWipeEmpty { + public string Id => "Ecryptfs"; public string DisplayName => "eCryptfs"; public FormatCategory Category => FormatCategory.Archive; public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract; + FormatCapabilities.CanList + | FormatCapabilities.CanExtract + | FormatCapabilities.CanCreate + | FormatCapabilities.CanModify + | FormatCapabilities.SupportsPassword; public string DefaultExtension => ".ecryptfs"; public IReadOnlyList Extensions => [".ecryptfs"]; public IReadOnlyList CompoundExtensions => []; - public IReadOnlyList MagicSignatures => [ - // eCryptfs marker: 0x3C81B7F5 big-endian at file offset 0. - new([0x3C, 0x81, 0xB7, 0xF5], Offset: 0, Confidence: 0.95), + /// + /// eCryptfs has no fixed byte magic: its marker is two random 32-bit words + /// whose XOR equals 0x3C81B7F5. Static signature matching cannot express that + /// relation, so detection falls back to the extension and the reader performs + /// the authoritative relational check. + /// + public IReadOnlyList MagicSignatures => []; + + public IReadOnlyList Methods => [ + new("aes128", "AES-128-CBC"), + new("aes192", "AES-192-CBC"), + new("aes256", "AES-256-CBC"), ]; - public IReadOnlyList Methods => [new("stored", "Stored")]; public string? TarCompressionFormatId => null; public AlgorithmFamily Family => AlgorithmFamily.Archive; - public string Description => "eCryptfs file-level encryption container — header surface + opaque ciphertext."; + public string Description => "Linux eCryptfs per-file lower-file encryption container — passphrase AES read/create/replace/remove."; public List List(Stream stream, string? password) { - var r = new EcryptfsReader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo( - i, e.Name, e.Size, e.Size, "Stored", e.IsDirectory, false, null)).ToList(); + using var reader = new EcryptfsReader(stream); + if (reader.DecryptedSize == 0) return []; + return [new ArchiveEntryInfo( + 0, + "content.bin", + checked((long)reader.DecryptedSize), + reader.CanonicalLength - reader.MetadataSize, + reader.CipherDescription, + IsDirectory: false, + IsEncrypted: true, + LastModified: null + )]; } public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new EcryptfsReader(stream); - foreach (var e in r.Entries) { - if (e.IsDirectory) continue; - if (files != null && !MatchesFilter(e.Name, files)) continue; - WriteFile(outputDir, e.Name, r.Extract(e)); + using var reader = new EcryptfsReader(stream); + if (reader.DecryptedSize == 0) return; + if (files != null && !MatchesFilter("content.bin", files)) return; + if (password is null) + throw new ArgumentException("A passphrase is required to extract an eCryptfs lower file.", nameof(password)); + WriteFile(outputDir, "content.bin", reader.ExtractContent(password)); + } + + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + ArgumentNullException.ThrowIfNull(options); + if (options.Password is null) + throw new ArgumentException("A passphrase is required to create an eCryptfs lower file.", nameof(options)); + + var files = inputs.Where(i => !i.IsDirectory).ToList(); + if (inputs.Any(i => i.IsDirectory)) + throw new NotSupportedException("An eCryptfs lower file represents one regular file; directories are not embedded in it."); + if (files.Count > 1) + throw new NotSupportedException("An eCryptfs lower file can contain exactly one logical file payload."); + + var content = files.Count == 0 ? Array.Empty() : files[0].ReadContent(); + var method = options.EncryptionMethod ?? options.MethodName; + EcryptfsCodec.Create(output, content, options.Password, EcryptfsCodec.ResolveKeySize(method)); + } + + /// + /// Credential-free mutation cannot safely replace encrypted plaintext. Callers + /// must use the + /// overload instead of relying on ambient or format-specific password state. + /// + void IArchiveModifiable.Add(Stream archive, IReadOnlyList inputs) + => throw new InvalidOperationException( + "eCryptfs replacement requires a passphrase; use the credential-aware IArchiveModifiable.Add overload."); + + /// + /// Replaces the one logical upper-file payload after validating the existing + /// lower file with the supplied passphrase. A complete replacement lower file + /// is staged before the caller's stream is changed, so a bad passphrase or + /// writer failure leaves the original bytes untouched. + /// + void IArchiveModifiable.Add( + Stream archive, + IReadOnlyList inputs, + ArchiveMutationOptions options + ) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + ArgumentNullException.ThrowIfNull(options); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("eCryptfs mutation requires a readable, writable, seekable stream.", nameof(archive)); + if (options.Password is null) + throw new ArgumentException("A passphrase is required to modify an eCryptfs lower file.", nameof(options)); + if (inputs.Any(i => i.IsDirectory)) + throw new NotSupportedException("An eCryptfs lower file represents one regular file; directories cannot be added."); + + var files = inputs.Where(i => !i.IsDirectory).ToList(); + if (files.Count != 1) + throw new NotSupportedException("eCryptfs replacement requires exactly one regular-file payload."); + + archive.Position = 0; + int keySize; + using (var reader = new EcryptfsReader(archive)) { + reader.ValidatePassword(options.Password); + keySize = reader.CipherDescription switch { + "AES-128-CBC" => 16, + "AES-192-CBC" => 24, + "AES-256-CBC" => 32, + _ => throw new NotSupportedException($"Cannot rewrite eCryptfs cipher '{reader.CipherDescription}'."), + }; } + + var content = files[0].ReadContent(); + using var staged = new MemoryStream(); + EcryptfsCodec.Create(staged, content, options.Password, keySize); + + staged.Position = 0; + archive.Position = 0; + archive.SetLength(0); + staged.CopyTo(archive); + archive.Position = 0; + } + + /// + /// Removes the single logical payload. This does not require the FEK: the + /// plaintext length can be set to zero and ciphertext extents discarded while + /// retaining valid authentication-token metadata. + /// + void IArchiveModifiable.Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(entryNames); + if (entryNames.Any(IsContentName)) + EcryptfsCodec.Purge(archive); + } + + void IArchiveModifiable.Remove(Stream archive, string[] entryNames, ArchiveMutationOptions options) { + ArgumentNullException.ThrowIfNull(options); + ((IArchiveModifiable)this).Remove(archive, entryNames); } + + /// + /// Clears metadata padding and bytes beyond the canonical encrypted extent set. + /// Ciphertext inside the final live extent is never touched: plaintext tail + /// zeroing would require the passphrase, which the wipe interface intentionally + /// does not carry. + /// + public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) + => EcryptfsCodec.WipeUnusedSpace(image); + + /// + /// Drops any bytes after the encrypted extents implied by the header. This is + /// the only meaningful compact/shrink operation for a single eCryptfs lower file. + /// + public void Shrink(Stream input, Stream output) => EcryptfsCodec.Shrink(input, output); + + /// + /// Removes the logical payload without requiring the FEK: set plaintext length + /// to zero and truncate all ciphertext extents while retaining valid key metadata. + /// + public void Purge(Stream archive) => EcryptfsCodec.Purge(archive); + + private static bool IsContentName(string name) + => Path.GetFileName(name.Replace('\\', '/')).Equals("content.bin", StringComparison.OrdinalIgnoreCase); } diff --git a/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs b/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs index b9fed5f75..c952de2f2 100644 --- a/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs +++ b/FileSystems/FileSystem.Ecryptfs/EcryptfsReader.cs @@ -1,127 +1,92 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; -using System.Globalization; -using System.Text; +using System.Security.Cryptography; namespace FileSystem.Ecryptfs; /// -/// Reads eCryptfs per-file encryption headers. eCryptfs is a stacking -/// file-level encryption filesystem (Linux) — every encrypted file is -/// stored on the lower filesystem as a regular file whose first page is -/// a metadata header followed by AES-CBC ciphertext extents. The header -/// starts with a 4-byte big-endian marker (0x3C81B7F5) so the -/// individual on-disk container is well-defined and detectable. -/// Decryption requires the user's mount passphrase + EFEK (Encrypted -/// File Encryption Key) tag-3 / tag-11 packets and is OUT OF SCOPE; this -/// reader surfaces the parsed header + the encrypted payload as a single -/// opaque entry. -/// -/// File header layout (big-endian, file offset 0): -/// 0x00 u32 marker == 0x3C81B7F5 -/// 0x04 u64 decrypted-size (plaintext length, host-endian on Linux) -/// 0x0C u32 flags -/// 0x10 u32 extent-size (typically 4096) -/// 0x14 ... EFEK packets, tag-3 / tag-11 OpenPGP-style -/// ~0x800 start of AES-CBC ciphertext extents +/// Reader for an eCryptfs lower file. eCryptfs is a stacked filesystem, so one +/// lower file represents one encrypted upper file rather than a complete volume. /// public sealed class EcryptfsReader : IDisposable { - private readonly byte[] _data; + private readonly Stream _stream; + private readonly EcryptfsCodec.Header _header; private readonly List _entries = []; - /// - /// Gets the entries. - /// - public IReadOnlyList Entries => _entries; - - /// - /// Gets or sets the marker. - /// - public uint Marker { get; private set; } - /// - /// Gets or sets the decrypted size. - /// - public ulong DecryptedSize { get; private set; } - /// - /// Gets or sets the flags. - /// - public uint Flags { get; private set; } - /// - /// Gets or sets the extent size. - /// - public uint ExtentSize { get; private set; } - /// - /// Gets a value indicating whether valid header. - /// - public bool ValidHeader { get; private set; } - - /// - /// Defines the ecryptfs marker constant value. - /// - public const uint EcryptfsMarker = 0x3C81B7F5u; - private const int HeaderMinSize = 24; - - /// - /// Initializes a new instance of . - /// public EcryptfsReader(Stream stream) { - using var ms = new MemoryStream(); - stream.CopyTo(ms); - _data = ms.ToArray(); - Parse(); - } + ArgumentNullException.ThrowIfNull(stream); + this._stream = stream; + this._header = EcryptfsCodec.ReadHeader(stream); - private void Parse() { - if (_data.Length < HeaderMinSize) - throw new InvalidDataException("Ecryptfs: file too small for header."); - - var marker = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(0, 4)); - if (marker != EcryptfsMarker) - throw new InvalidDataException($"Ecryptfs: invalid marker 0x{marker:X8} at offset 0 (expected 0x{EcryptfsMarker:X8})."); - - this.Marker = marker; - // Decrypted-size is 8 bytes BE per the eCryptfs on-disk format documentation - // (kernel writes it big-endian regardless of host byte order so files round-trip across hosts). - this.DecryptedSize = BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(4, 8)); - this.Flags = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(12, 4)); - this.ExtentSize = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(16, 4)); - this.ValidHeader = true; - - var meta = BuildMetadata(); - _entries.Add(new EcryptfsEntry { Name = "FULL.ecryptfs", Size = _data.Length, IsDirectory = false, Data = _data }); - _entries.Add(new EcryptfsEntry { Name = "metadata.ini", Size = meta.Length, IsDirectory = false, Data = meta }); - - // Surface the encrypted ciphertext as opaque blob (everything after the metadata page). - var payloadStart = (int)Math.Min((long)Math.Max(this.ExtentSize, 4096), _data.Length); - var payloadLen = _data.Length - payloadStart; - if (payloadLen > 0) { - var blob = _data.AsSpan(payloadStart, payloadLen).ToArray(); - _entries.Add(new EcryptfsEntry { Name = "ciphertext.bin", Size = blob.Length, IsDirectory = false, Data = blob }); + if (this._header.DecryptedSize > 0) { + if (this._header.DecryptedSize > long.MaxValue) + throw new InvalidDataException("eCryptfs plaintext size exceeds the supported entry-size range."); + this._entries.Add(new EcryptfsEntry { + Name = "content.bin", + Size = (long)this._header.DecryptedSize, + IsDirectory = false, + }); } } - private byte[] BuildMetadata() { - var bldr = new StringBuilder(); - bldr.Append("parse_status=ok\n"); - bldr.Append("format=eCryptfs (file-level encryption)\n"); - bldr.Append(CultureInfo.InvariantCulture, $"marker=0x{this.Marker:X8}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"decrypted_size={this.DecryptedSize}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"flags=0x{this.Flags:X8}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"extent_size={this.ExtentSize}\n"); - bldr.Append("note=Ciphertext exposed opaque; decryption requires mount passphrase + EFEK.\n"); - return Encoding.UTF8.GetBytes(bldr.ToString()); - } + /// The validated eCryptfs marker relation constant. + public uint Marker => EcryptfsCodec.MarkerMagic; - /// - /// Decodes the supplied input. - /// - public byte[] Extract(EcryptfsEntry entry) { - ArgumentNullException.ThrowIfNull(entry); - return entry.Data; + /// The random first word of the two-word marker pair stored on disk. + public uint MarkerWord => this._header.MarkerWord; + + /// Plaintext file length stored in the lower-file header. + public ulong DecryptedSize => this._header.DecryptedSize; + + /// Raw eCryptfs file flags, including the version in the top byte. + public uint Flags => this._header.Flags; + + /// eCryptfs lower-file format version. + public byte FileVersion => this._header.FileVersion; + + /// Encryption extent size in bytes. + public uint ExtentSize => this._header.ExtentSize; + + /// Number of header extents preceding ciphertext data. + public ushort HeaderExtentCount => this._header.HeaderExtentCount; + + /// Total metadata region at the beginning of the lower file. + public int MetadataSize => this._header.MetadataSize; + + /// Canonical lower-file length implied by the plaintext size and extent geometry. + public long CanonicalLength => this._header.CanonicalLength; + + /// Human-readable cipher of the first passphrase packet, when present. + public string CipherDescription => this._header.CipherDescription; + + /// Raw eight-byte authentication-token signature as lowercase hexadecimal. + public string? PassphraseSignature + => this._header.PassphrasePackets.Count == 0 + ? null + : Convert.ToHexStringLower(this._header.PassphrasePackets[0].Signature); + + public IReadOnlyList Entries => this._entries; + + /// Decrypts the single logical upper-file payload with a passphrase. + /// The passphrase does not match this lower file. + public byte[] ExtractContent(string password) { + if (this._header.DecryptedSize > int.MaxValue) + throw new IOException("This eCryptfs payload is too large for the in-memory extraction API."); + using var output = new MemoryStream((int)this._header.DecryptedSize); + EcryptfsCodec.DecryptTo(this._stream, output, this._header, password); + return output.ToArray(); } /// - /// Releases resources held by this instance. + /// Validates a passphrase against the authentication-token packet without + /// materializing plaintext in memory. The ciphertext is streamed through the + /// normal decoder into , so the same key and extent + /// path used by extraction is exercised before a destructive mutation starts. /// - public void Dispose() { } + /// The passphrase does not match this lower file. + public void ValidatePassword(string password) + => EcryptfsCodec.DecryptTo(this._stream, Stream.Null, this._header, password); + + public void Dispose() { + // The reader never owns the caller's stream. + } } diff --git a/FileSystems/FileSystem.Ext/ExtExtendedAttributes.cs b/FileSystems/FileSystem.Ext/ExtExtendedAttributes.cs new file mode 100644 index 000000000..904a58f0c --- /dev/null +++ b/FileSystems/FileSystem.Ext/ExtExtendedAttributes.cs @@ -0,0 +1,403 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; +using Compression.Core.DiskImage; + +namespace FileSystem.Ext; + +/// +/// Reads ext2/3/4 extended attributes and performs conservative in-inode mutations. +/// +/// +/// +/// ext stores small xattrs after the extended inode fields and may store additional +/// attributes in one external xattr block referenced by i_file_acl. Both forms +/// are readable here. Mutations deliberately target only the in-inode form: growing +/// or rewriting an external xattr block also requires block allocation/reference-count +/// and metadata-checksum publication, so unsupported cases fail closed. +/// +/// +/// The implementation follows the Linux ext4 on-disk documentation. No kernel +/// implementation code is copied. +/// +/// +public static class ExtExtendedAttributes { + private const int SuperblockOffset = 1024; + private const ushort ExtMagic = 0xEF53; + private const uint XattrMagic = 0xEA020000; + private const uint Incompat64Bit = 0x0080; + private const uint IncompatCsumSeed = 0x2000; + private const uint RoCompatMetadataCsum = 0x0400; + private const int ClassicInodeSize = 128; + private const int EntryHeaderSize = 16; + + /// Reads all directly stored xattrs for . + public static IReadOnlyDictionary Read(Stream image, string path) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + EnsureReadableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inodeOffset = geometry.InodeOffset(image, inodeNumber); + var inode = ReadAt(image, inodeOffset, geometry.InodeSize); + var result = new Dictionary(StringComparer.Ordinal); + + ReadInline(inode, result); + + var externalBlock = ExternalXattrBlock(inode, geometry.Has64Bit); + if (externalBlock != 0) { + var blockOffset = checked((long)externalBlock * geometry.BlockSize); + if (blockOffset < 0 || blockOffset + geometry.BlockSize > image.Length) + throw new InvalidDataException("ext: external xattr block lies outside the image."); + var block = ReadAt(image, blockOffset, geometry.BlockSize); + if (BinaryPrimitives.ReadUInt32LittleEndian(block) != XattrMagic) + throw new InvalidDataException("ext: i_file_acl does not reference an ext xattr block."); + ReadEntries(block, EntryHeaderOffset: 32, ValueBaseOffset: 0, result); + } + + return result; + } finally { + image.Position = original; + } + } + + /// + /// Creates or replaces one in-inode xattr. External-block and EA-inode values + /// are never rewritten by this conservative mutator. + /// + public static void Set(Stream image, string path, string name, ReadOnlySpan value) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + EnsureWritableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inodeOffset = geometry.InodeOffset(image, inodeNumber); + var inode = ReadAt(image, inodeOffset, geometry.InodeSize); + + var externalBlock = ExternalXattrBlock(inode, geometry.Has64Bit); + if (externalBlock != 0) { + var external = ReadExternal(image, geometry, externalBlock); + if (external.ContainsKey(name)) + throw new NotSupportedException( + $"ext: xattr '{name}' is stored in an external xattr block; external-block mutation is intentionally unsupported."); + } + + var attributes = ReadInlineDictionary(inode); + attributes[name] = value.ToArray(); + WriteInline(inode, attributes); + StampInodeIfNeeded(geometry, inodeNumber, inode); + WriteAt(image, inodeOffset, inode); + image.Flush(); + } finally { + image.Position = original; + } + } + + /// Removes one in-inode xattr, returning false when it was absent. + public static bool Remove(Stream image, string path, string name) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + EnsureWritableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inodeOffset = geometry.InodeOffset(image, inodeNumber); + var inode = ReadAt(image, inodeOffset, geometry.InodeSize); + + var attributes = ReadInlineDictionary(inode); + if (attributes.Remove(name)) { + WriteInline(inode, attributes); + StampInodeIfNeeded(geometry, inodeNumber, inode); + WriteAt(image, inodeOffset, inode); + image.Flush(); + return true; + } + + var externalBlock = ExternalXattrBlock(inode, geometry.Has64Bit); + if (externalBlock != 0 && ReadExternal(image, geometry, externalBlock).ContainsKey(name)) + throw new NotSupportedException( + $"ext: xattr '{name}' is stored in an external xattr block; external-block mutation is intentionally unsupported."); + + return false; + } finally { + image.Position = original; + } + } + + private static Dictionary ReadInlineDictionary(byte[] inode) { + var result = new Dictionary(StringComparer.Ordinal); + ReadInline(inode, result); + return result; + } + + private static void ReadInline(byte[] inode, Dictionary result) { + if (inode.Length <= ClassicInodeSize) return; + var extraIsize = BinaryPrimitives.ReadUInt16LittleEndian(inode.AsSpan(ClassicInodeSize)); + var headerOffset = ClassicInodeSize + extraIsize; + if (headerOffset < ClassicInodeSize || headerOffset + 4 > inode.Length) return; + if (BinaryPrimitives.ReadUInt32LittleEndian(inode.AsSpan(headerOffset)) != XattrMagic) return; + + // For inode-body xattrs e_value_offs is relative to the first xattr entry, + // immediately after the four-byte ibody header. + var firstEntry = headerOffset + 4; + ReadEntries(inode, firstEntry, firstEntry, result); + } + + private static Dictionary ReadExternal(Stream image, Geometry geometry, ulong blockNumber) { + var offset = checked((long)blockNumber * geometry.BlockSize); + if (offset < 0 || offset + geometry.BlockSize > image.Length) + throw new InvalidDataException("ext: external xattr block lies outside the image."); + var block = ReadAt(image, offset, geometry.BlockSize); + if (BinaryPrimitives.ReadUInt32LittleEndian(block) != XattrMagic) + throw new InvalidDataException("ext: invalid external xattr block magic."); + var result = new Dictionary(StringComparer.Ordinal); + ReadEntries(block, 32, 0, result); + return result; + } + + private static void ReadEntries( + byte[] storage, + int EntryHeaderOffset, + int ValueBaseOffset, + Dictionary result) { + var pos = EntryHeaderOffset; + while (pos + EntryHeaderSize <= storage.Length) { + // Four zero bytes terminate the entry array. + if (BinaryPrimitives.ReadUInt32LittleEndian(storage.AsSpan(pos)) == 0) break; + + var nameLength = storage[pos]; + var nameIndex = storage[pos + 1]; + var valueOffset = BinaryPrimitives.ReadUInt16LittleEndian(storage.AsSpan(pos + 2)); + var valueInode = BinaryPrimitives.ReadUInt32LittleEndian(storage.AsSpan(pos + 4)); + var valueSize = BinaryPrimitives.ReadUInt32LittleEndian(storage.AsSpan(pos + 8)); + var entryLength = Align4(EntryHeaderSize + nameLength); + if (nameLength == 0 || pos + entryLength > storage.Length) + throw new InvalidDataException("ext: malformed xattr entry."); + if (valueInode != 0) + throw new NotSupportedException("ext: EA-inode xattr values are not supported by the image accessor."); + if (valueSize > int.MaxValue) + throw new InvalidDataException("ext: xattr value is too large."); + + var valueAt = checked(ValueBaseOffset + valueOffset); + if (valueAt < 0 || valueAt + (long)valueSize > storage.Length) + throw new InvalidDataException("ext: xattr value lies outside its storage region."); + + var suffix = Encoding.UTF8.GetString(storage, pos + EntryHeaderSize, nameLength); + var fullName = ExpandName(nameIndex, suffix); + var value = storage.AsSpan(valueAt, (int)valueSize).ToArray(); + result[fullName] = value; + pos += entryLength; + } + } + + private static void WriteInline(byte[] inode, IReadOnlyDictionary attributes) { + if (inode.Length <= ClassicInodeSize) + throw new NotSupportedException("ext: 128-byte inodes have no in-inode xattr area."); + + var extraIsize = BinaryPrimitives.ReadUInt16LittleEndian(inode.AsSpan(ClassicInodeSize)); + if (extraIsize == 0) + throw new NotSupportedException("ext: inode does not declare an extended inode area for inline xattrs."); + + var headerOffset = ClassicInodeSize + extraIsize; + if (headerOffset < ClassicInodeSize || headerOffset + 4 > inode.Length) + throw new InvalidDataException("ext: invalid i_extra_isize leaves no xattr body header."); + + var region = inode.AsSpan(headerOffset); + region.Clear(); + if (attributes.Count == 0) return; + + BinaryPrimitives.WriteUInt32LittleEndian(region, XattrMagic); + var firstEntry = headerOffset + 4; + var entryPos = firstEntry; + var valuePos = inode.Length; + + foreach (var pair in attributes.OrderBy(pair => pair.Key, StringComparer.Ordinal)) { + var (nameIndex, suffix) = CompressName(pair.Key); + var nameBytes = Encoding.UTF8.GetBytes(suffix); + if (nameBytes.Length is 0 or > byte.MaxValue) + throw new ArgumentException($"ext: xattr name '{pair.Key}' is not representable."); + var value = pair.Value ?? []; + + valuePos = AlignDown4(valuePos - value.Length); + var entryLength = Align4(EntryHeaderSize + nameBytes.Length); + if (entryPos + entryLength + 4 > valuePos) + throw new NotSupportedException( + $"ext: xattrs do not fit in inode {inode.Length}-byte inline storage; external-block allocation is not implemented."); + + var relativeValueOffset = valuePos - firstEntry; + if ((uint)relativeValueOffset > ushort.MaxValue) + throw new NotSupportedException("ext: inline xattr value offset exceeds the on-disk field width."); + + inode[entryPos] = (byte)nameBytes.Length; + inode[entryPos + 1] = nameIndex; + BinaryPrimitives.WriteUInt16LittleEndian(inode.AsSpan(entryPos + 2), (ushort)relativeValueOffset); + BinaryPrimitives.WriteUInt32LittleEndian(inode.AsSpan(entryPos + 4), 0); // e_value_inum + BinaryPrimitives.WriteUInt32LittleEndian(inode.AsSpan(entryPos + 8), checked((uint)value.Length)); + BinaryPrimitives.WriteUInt32LittleEndian(inode.AsSpan(entryPos + 12), 0); // e_hash optional for ibody attrs + nameBytes.CopyTo(inode.AsSpan(entryPos + EntryHeaderSize)); + value.CopyTo(inode.AsSpan(valuePos)); + entryPos += entryLength; + } + + // Four zero bytes after the final entry terminate the entry array; the region + // was cleared before encoding, so no additional write is needed. + } + + private static ulong ExternalXattrBlock(ReadOnlySpan inode, bool has64Bit) { + if (inode.Length < 108) return 0; + var low = BinaryPrimitives.ReadUInt32LittleEndian(inode[0x68..]); + ulong high = 0; + // Linux ext4 inode osd2 starts at 0x74; l_i_file_acl_high is the u16 at 0x76. + if (has64Bit && inode.Length >= 0x78) + high = BinaryPrimitives.ReadUInt16LittleEndian(inode[0x76..]); + return low | high << 32; + } + + private static string ExpandName(byte index, string suffix) => index switch { + 1 => "user." + suffix, + 2 => "system.posix_acl_access" + suffix, + 3 => "system.posix_acl_default" + suffix, + 4 => "trusted." + suffix, + 6 => "security." + suffix, + 7 => "system." + suffix, + _ => $"ext.index{index}." + suffix, + }; + + private static (byte Index, string Suffix) CompressName(string name) { + if (name.StartsWith("user.", StringComparison.Ordinal)) return (1, name[5..]); + if (name.Equals("system.posix_acl_access", StringComparison.Ordinal)) return (2, ""); + if (name.Equals("system.posix_acl_default", StringComparison.Ordinal)) return (3, ""); + if (name.StartsWith("trusted.", StringComparison.Ordinal)) return (4, name[8..]); + if (name.StartsWith("security.", StringComparison.Ordinal)) return (6, name[9..]); + if (name.StartsWith("system.", StringComparison.Ordinal)) return (7, name[7..]); + throw new ArgumentException( + $"ext: xattr '{name}' has no supported namespace prefix (user., trusted., security., system.).", + nameof(name)); + } + + private static uint ResolveInode(Stream image, string path, uint rootInode) { + var normalized = Normalize(path); + if (normalized.Length == 0) return rootInode; + image.Position = 0; + using var reader = new ExtReader(image, leaveOpen: true); + var entry = reader.Entries.FirstOrDefault(entry => + string.Equals(Normalize(entry.Name), normalized, StringComparison.Ordinal)); + return entry?.Inode ?? throw new FileNotFoundException($"ext entry not found: {path}"); + } + + private static string Normalize(string path) => path.Replace('\\', '/').Trim('/'); + + private static void StampInodeIfNeeded(Geometry geometry, uint inodeNumber, byte[] inode) { + if (!geometry.HasMetadataChecksum) return; + var generation = inode.Length >= 0x68 + ? BinaryPrimitives.ReadUInt32LittleEndian(inode.AsSpan(0x64)) + : 0; + var inodeSeed = ExtChecksums.InodeSeed(geometry.ChecksumSeed, inodeNumber, generation); + ExtChecksums.StampInode(inode, geometry.InodeSize, inodeSeed); + } + + private static byte[] ReadAt(Stream image, long offset, int length) { + var data = new byte[length]; + image.Position = offset; + image.ReadExactly(data); + return data; + } + + private static void WriteAt(Stream image, long offset, ReadOnlySpan data) { + image.Position = offset; + image.Write(data); + } + + private static int Align4(int value) => checked((value + 3) & ~3); + private static int AlignDown4(int value) => value & ~3; + + private static void EnsureReadableSeekable(Stream image) { + if (!image.CanRead || !image.CanSeek) + throw new ArgumentException("ext xattr access requires a readable, seekable image.", nameof(image)); + } + + private static void EnsureWritableSeekable(Stream image) { + EnsureReadableSeekable(image); + if (!image.CanWrite) + throw new ArgumentException("ext xattr mutation requires a writable image.", nameof(image)); + } + + private readonly record struct Geometry( + int BlockSize, + uint InodesPerGroup, + int InodeSize, + uint FirstDataBlock, + int DescriptorSize, + bool Has64Bit, + bool HasMetadataChecksum, + uint ChecksumSeed, + uint RootInode) { + + public static Geometry Read(Stream image) { + if (image.Length < SuperblockOffset + 1024) + throw new InvalidDataException("ext: image is too small for its superblock."); + var sb = ReadAt(image, SuperblockOffset, 1024); + if (BinaryPrimitives.ReadUInt16LittleEndian(sb.AsSpan(56)) != ExtMagic) + throw new InvalidDataException("ext: invalid superblock magic."); + + var blockSize = checked(1024 << (int)BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(24))); + var inodesPerGroup = BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(40)); + var inodeSize = BinaryPrimitives.ReadUInt16LittleEndian(sb.AsSpan(88)); + if (inodeSize == 0) inodeSize = ClassicInodeSize; + var firstDataBlock = BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(20)); + var featureIncompat = BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(96)); + var featureRoCompat = BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(100)); + var descriptorSize = ExtBlockGroupGeometry.DescriptorSize(sb); + var checksumSeed = (featureIncompat & IncompatCsumSeed) != 0 + ? BinaryPrimitives.ReadUInt32LittleEndian(sb.AsSpan(624)) + : ExtChecksums.SeedFromUuid(sb.AsSpan(104, 16)); + + if (blockSize is < 1024 or > 65536 || (blockSize & (blockSize - 1)) != 0) + throw new InvalidDataException($"ext: invalid block size {blockSize}."); + if (inodesPerGroup == 0) + throw new InvalidDataException("ext: superblock declares zero inodes per group."); + if (inodeSize < ClassicInodeSize) + throw new InvalidDataException($"ext: invalid inode size {inodeSize}."); + + return new Geometry( + blockSize, + inodesPerGroup, + inodeSize, + firstDataBlock, + descriptorSize, + (featureIncompat & Incompat64Bit) != 0, + (featureRoCompat & RoCompatMetadataCsum) != 0, + checksumSeed, + RootInode: 2); + } + + public long InodeOffset(Stream image, uint inodeNumber) { + if (inodeNumber == 0) + throw new ArgumentOutOfRangeException(nameof(inodeNumber)); + var group = (inodeNumber - 1) / this.InodesPerGroup; + var index = (inodeNumber - 1) % this.InodesPerGroup; + var bgdtBlock = this.FirstDataBlock + 1; + var descriptorOffset = checked((long)bgdtBlock * this.BlockSize + (long)group * this.DescriptorSize); + if (descriptorOffset + this.DescriptorSize > image.Length) + throw new InvalidDataException("ext: inode group descriptor lies outside the image."); + var descriptor = ReadAt(image, descriptorOffset, this.DescriptorSize); + ulong tableBlock = BinaryPrimitives.ReadUInt32LittleEndian(descriptor.AsSpan(8)); + if (this.Has64Bit && descriptor.Length >= 44) + tableBlock |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(descriptor.AsSpan(40)) << 32; + var offset = checked((long)tableBlock * this.BlockSize + (long)index * this.InodeSize); + if (offset < 0 || offset + this.InodeSize > image.Length) + throw new InvalidDataException("ext: inode lies outside the image."); + return offset; + } + } +} diff --git a/FileSystems/FileSystem.GlusterFs/FileSystem.GlusterFs.csproj b/FileSystems/FileSystem.GlusterFs/FileSystem.GlusterFs.csproj index 78cd2b68f..24a7d9e51 100644 --- a/FileSystems/FileSystem.GlusterFs/FileSystem.GlusterFs.csproj +++ b/FileSystems/FileSystem.GlusterFs/FileSystem.GlusterFs.csproj @@ -4,5 +4,7 @@ + + diff --git a/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs b/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs index 73195b9d2..936f723d6 100644 --- a/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs +++ b/FileSystems/FileSystem.GlusterFs/GlusterFsEntry.cs @@ -2,7 +2,7 @@ namespace FileSystem.GlusterFs; /// -/// Represents a gluster fs entry. +/// Represents an entry surfaced from a single GlusterFS brick backing store. /// public sealed class GlusterFsEntry { /// @@ -22,7 +22,17 @@ public sealed class GlusterFsEntry { /// public long Offset { get; init; } /// - /// Gets or sets the data. + /// Gets or sets eagerly materialized data, used by synthetic metadata entries. /// public byte[] Data { get; init; } = []; + + /// + /// Lazily materializes an entry delegated to the backing filesystem reader. + /// + internal Func? DataFactory { get; init; } + + /// + /// Native path inside the backing filesystem, used for xattr lookup. + /// + internal string? BackingPath { get; init; } } diff --git a/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs b/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs index f99f78848..38eebd78a 100644 --- a/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs +++ b/FileSystems/FileSystem.GlusterFs/GlusterFsFormatDescriptor.cs @@ -1,39 +1,58 @@ #pragma warning disable CS1591 +using System.Buffers.Binary; using Compression.Registry; using Compression.Registry.Streaming; +using FileSystem.Ext; using static Compression.Registry.FormatHelpers; namespace FileSystem.GlusterFs; /// -/// Stage 0 detection-only descriptor for GlusterFS. Honest fallback: -/// GlusterFS has no on-disk image format. A GlusterFS volume is a -/// logical aggregation of one or more "bricks", and every brick is -/// just a normal directory on a local POSIX filesystem (typically -/// XFS or ext4). Volume files live at their normal POSIX paths inside -/// the brick directory and carry GlusterFS state in extended -/// attributes (trusted.gfid, trusted.glusterfs.dht, -/// trusted.glusterfs.volume-id, trusted.glusterfs.pathinfo, -/// etc.). There is no superblock, no brick header, no portable -/// single-file representation that this image-based pipeline can -/// consume. +/// Read-only single-brick view for GlusterFS backing-store images, with a +/// conservative backing-store shrink operation where it is provably metadata-safe. /// -/// We therefore stay Stage 0 permanently. The 0xCAFE5BAB magic -/// recognised here is a workbench-internal convention for hand-dumped -/// brick-object probes — it is not a real on-disk GlusterFS structure -/// and no real GlusterFS deployment will produce it. Promotion to R/O -/// would require walking a live directory tree and reading xattrs, -/// which is outside the image-stream contract enforced by -/// . +/// GlusterFS has no independent block format: a volume is a logical +/// collection of bricks and each brick is an export directory on an ordinary +/// filesystem with extended-attribute support. This descriptor therefore +/// delegates real backing images to the repository's XFS or ext2/3/4 reader and +/// surfaces the physical contents of one brick. It does not reconstruct the +/// distributed Gluster namespace. +/// +/// Detection is extension-only. XFS and ext images already belong to their +/// native descriptors and there is no Gluster-specific superblock magic with +/// which to distinguish a brick image automatically. The former workbench-only +/// 0xCAFE5BAB probe convention is deliberately not recognised as GlusterFS. +/// +/// The native XFS/ext layers can now read Gluster xattrs and conservatively +/// mutate the common short-form/in-inode storage cases. General archive mutation +/// remains disabled until every xattr storage form that a maintenance operation can +/// encounter (including ext external blocks/EA inodes and XFS leaf/btree/remote +/// attributes) is preserved. Advertising a rebuild-based mutation before then +/// could silently discard trusted.gfid or trusted.glusterfs.* state. +/// +/// Shrink is the deliberate exception. For ext-backed bricks the native +/// in-place shrinker chooses its boundary from the filesystem allocation bitmap; +/// every allocated xattr block therefore pins the boundary just like file data and +/// is preserved byte-for-byte. XFS-backed bricks copy through unchanged because the +/// current XFS shrink path may rebuild the image. This is a backing-store geometry +/// operation only; it is not Gluster's remove-brick command. +/// +/// Gluster volume operations such as rebalance, fix-layout, and remove-brick +/// are explicitly outside this single-image abstraction. They coordinate multiple +/// bricks and belong to a live cluster/volume control plane, not an offline brick +/// image editor. /// /// References: /// -/// https://docs.gluster.org — official GlusterFS documentation (brick/xattr architecture) -/// https://github.com/gluster/glusterfs — canonical source -/// https://en.wikipedia.org/wiki/GlusterFS — Wikipedia overview +/// https://docs.gluster.org/en/latest/Administrator-Guide/Setting-Up-Volumes/ — a volume is a logical collection of export-directory bricks +/// https://docs.gluster.org/en/latest/Administrator-Guide/GlusterFS-Introduction/ — backing filesystems must support extended attributes +/// https://docs.gluster.org/en/latest/Administrator-Guide/Managing-Volumes/ — remove-brick/rebalance/fix-layout are volume operations +/// https://docs.kernel.org/filesystems/ext4/attributes.html — ext4 xattr on-disk layout +/// https://www.kernel.org/pub/linux/utils/fs/xfs/docs/xfs_filesystem_structure.pdf — XFS short-form attribute layout +/// https://github.com/gluster/glusterfs — canonical GlusterFS implementation, dual GPLv2/LGPLv3+ /// /// -public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { +public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveShrinkable { /// /// Gets the id. @@ -42,7 +61,7 @@ public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveForma /// /// Gets the display name. /// - public string DisplayName => "GlusterFS"; + public string DisplayName => "GlusterFS brick"; /// /// Gets the category. /// @@ -51,7 +70,8 @@ public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveForma /// Gets the capabilities. /// public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest; + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | + FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories; /// /// Gets the default extension. /// @@ -65,14 +85,9 @@ public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveForma /// public IReadOnlyList CompoundExtensions => []; /// - /// Gets the magic signatures. + /// Gets the magic signatures. Empty by design because GlusterFS has no separate on-disk magic. /// - public IReadOnlyList MagicSignatures => [ - // 0xCAFE5BAB — workbench-internal convention for hand-dumped brick - // object probes, NOT a real on-disk GlusterFS marker. GlusterFS - // itself has no on-disk header. See type-level doc. - new([0xCA, 0xFE, 0x5B, 0xAB], Offset: 0, Confidence: 0.90), - ]; + public IReadOnlyList MagicSignatures => []; /// /// Gets the methods. /// @@ -89,41 +104,94 @@ public sealed class GlusterFsFormatDescriptor : IFormatDescriptor, IArchiveForma /// Gets the description. /// public string Description => - "GlusterFS — Stage 0 (detection-only, permanent). A GlusterFS brick is a normal " + - "directory on a local POSIX filesystem (XFS/ext4); files live at their POSIX paths " + - "and metadata lives in xattrs (trusted.gfid, trusted.glusterfs.*). There is no " + - "on-disk image format, so no R/O promotion is possible from a single image stream. " + - "The 0xCAFE5BAB magic is a workbench-internal probe convention, not a real " + - "GlusterFS marker."; + "GlusterFS single-brick R/O backing-store view via XFS/ext delegation. Native backing " + + "accessors read trusted.gfid/trusted.glusterfs.* and mutate inline/short-form xattrs. " + + "General writes, defrag, wipe, layout and purge stay disabled until all xattr storage " + + "forms are safe. Shrink-to-fit is supported for ext-backed bricks via the allocation " + + "bitmap; XFS-backed bricks copy through unchanged. Cluster rebalance, fix-layout and " + + "remove-brick are intentionally out of scope."; /// - /// Lists the entries in the supplied container. + /// Lists the entries in the supplied brick backing-store image. /// public List List(Stream stream, string? password) { - var r = new GlusterFsReader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo( - i, e.Name, e.Size, e.Size, "Stored", e.IsDirectory, false, null)).ToList(); + using var reader = new GlusterFsReader(stream); + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo( + index, entry.Name, entry.Size, entry.Size, "Stored", entry.IsDirectory, false, null)).ToList(); } /// - /// Decodes the supplied input. + /// Extracts entries from the supplied single-brick physical view. /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new GlusterFsReader(stream); - foreach (var e in r.Entries) { - if (e.IsDirectory) continue; - if (files != null && !MatchesFilter(e.Name, files)) continue; - WriteFile(outputDir, e.Name, r.Extract(e)); + using var reader = new GlusterFsReader(stream); + foreach (var entry in reader.Entries) { + if (entry.IsDirectory) continue; + if (files != null && !MatchesFilter(entry.Name, files)) continue; + WriteFile(outputDir, entry.Name, reader.Extract(entry)); } } Stream IArchiveFormatOperations.OpenEntry(Stream archive, string entryName, string? password) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(entryName); - var r = new GlusterFsReader(archive); - var entry = r.Entries.FirstOrDefault(e => e.Name == entryName) - ?? throw new FileNotFoundException($"GlusterFS entry not found: {entryName}"); - var data = r.Extract(entry); + using var reader = new GlusterFsReader(archive); + var entry = reader.Entries.FirstOrDefault(candidate => candidate.Name == entryName) + ?? throw new FileNotFoundException($"GlusterFS brick entry not found: {entryName}"); + var data = reader.Extract(entry); return new BoundedEntryStream(new MemoryStream(data, writable: false), data.Length, leaveOpen: false); } + + /// + /// Shrinks an ext-backed brick to its highest allocated block without rebuilding + /// file or xattr metadata. XFS-backed bricks are copied unchanged. + /// + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("GlusterFS shrink requires a readable, seekable input.", nameof(input)); + if (!output.CanRead || !output.CanWrite || !output.CanSeek) + throw new ArgumentException("GlusterFS shrink requires a readable, writable, seekable output.", nameof(output)); + + Stream target; + if (ReferenceEquals(input, output)) { + target = output; + } else { + input.Position = 0; + output.Position = 0; + output.SetLength(0); + input.CopyTo(output); + output.Flush(); + target = output; + } + + if (!IsExtBacking(target)) { + target.Position = 0; + return; + } + + target.Position = 0; + try { + ExtInPlaceShrinker.ShrinkToFit(target); + } catch (NotSupportedException) { + // Some valid ext geometries cannot be reduced by the conservative in-place + // shrinker. Leaving the already-copied image unchanged is the safe result. + } + target.Position = 0; + } + + private static bool IsExtBacking(Stream image) { + const long magicOffset = 1024 + 56; + if (image.Length < magicOffset + sizeof(ushort)) return false; + var original = image.Position; + Span magic = stackalloc byte[sizeof(ushort)]; + try { + image.Position = magicOffset; + image.ReadExactly(magic); + return BinaryPrimitives.ReadUInt16LittleEndian(magic) == 0xEF53; + } finally { + image.Position = original; + } + } } diff --git a/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs b/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs index bb3342d34..7ae715059 100644 --- a/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs +++ b/FileSystems/FileSystem.GlusterFs/GlusterFsReader.cs @@ -2,126 +2,290 @@ using System.Buffers.Binary; using System.Globalization; using System.Text; +using FileSystem.Ext; +using FileSystem.Xfs; namespace FileSystem.GlusterFs; /// -/// Stage 0 detection-only reader for GlusterFS — permanent honest -/// fallback. GlusterFS itself has no on-disk image format: a -/// brick is a normal directory on a local POSIX filesystem -/// (XFS / ext4 / ...) and volume files are stored at their normal -/// POSIX paths inside that directory. All GlusterFS-specific state -/// lives in extended attributes (the trusted.gfid, -/// trusted.glusterfs.dht, trusted.glusterfs.volume-id, -/// trusted.glusterfs.pathinfo namespace). +/// Reads a single GlusterFS brick backing-store image by delegating the native +/// on-disk filesystem to the existing XFS or ext2/3/4 reader. /// -/// Consequences: -/// -/// There is no superblock or brick header to parse. -/// Distribution / replication state (DHT hashing → brick -/// mapping, AFR replicate metadata, EC dispersed metadata, rebalance -/// bookkeeping) only exists across multiple bricks on multiple -/// hosts, not inside any single image. -/// An R/O promotion is fundamentally incompatible with this -/// project's image-stream contract — recognising a GlusterFS -/// "volume" would require walking a live POSIX directory tree and -/// reading xattrs through the host OS, which is outside the -/// Stream-based IArchiveFormatOperations surface. -/// +/// GlusterFS does not define a separate block format. A brick is an export +/// directory on an ordinary filesystem that supports extended attributes. The +/// logical volume namespace and DHT/AFR/EC state span multiple bricks, while +/// object identity and translator metadata are stored in xattrs such as +/// trusted.gfid and trusted.glusterfs.*. Consequently this reader +/// intentionally exposes only the physical view of one supplied backing image. +/// It does not claim to reconstruct a Gluster volume from one brick. /// -/// The 0xCAFE5BAB magic verified by is a -/// workbench-internal probe convention used to dump and round-trip -/// hand-crafted "brick object" experiments; it is not a real -/// on-disk GlusterFS marker and no real GlusterFS deployment produces -/// it. The reader therefore stays a thin two-entry detector -/// (synthetic metadata.ini + raw gluster-brick.bin) and -/// will never grow real semantics. +/// When the backing namespace contains Gluster's .glusterfs GFID +/// index, its parent directory identifies the brick root. Entries outside that +/// subtree and the index itself are omitted from the normal view. If no index is +/// present, the filesystem root is used as a conservative explicit-input +/// fallback. +/// +/// Gluster xattrs are readable through . +/// The backing accessors also support conservative inline/short-form mutation, +/// but this brick reader remains read-only until all xattr storage forms needed +/// by a maintenance operation can be preserved without rebuilding metadata. /// public sealed class GlusterFsReader : IDisposable { - /// - /// Workbench-internal probe magic (0xCA 0xFE 0x5B 0xAB, 0xCAFE5BAB - /// big-endian) used by the detector tests. Not a real - /// GlusterFS structure — GlusterFS has no on-disk header at all. - /// - public static readonly byte[] BrickMagic = [0xCA, 0xFE, 0x5B, 0xAB]; - - private const int HeaderSize = 8; + private const int ExtSuperblockOffset = 1024; + private const int ExtMagicOffset = ExtSuperblockOffset + 56; + private const ushort ExtMagic = 0xEF53; + private const string GlusterIndexName = ".glusterfs"; - private readonly byte[] _data; + private readonly Stream _image; + private readonly bool _ownsImage; private readonly List _entries = []; + private XfsReader? _xfsReader; + private ExtReader? _extReader; /// - /// Gets the entries. + /// Gets the entries in the single-brick physical view. /// public IReadOnlyList Entries => _entries; + + /// + /// Gets a value indicating whether a supported backing filesystem was found. + /// + public bool ValidHeader { get; private set; } + /// - /// Gets or sets the magic word. + /// Gets the detected backing filesystem name (xfs or ext). /// - public uint MagicWord { get; private set; } + public string BackingFileSystem { get; private set; } = ""; + /// - /// Gets or sets the trailing word. + /// Gets the inferred brick root inside the backing filesystem. Empty means filesystem root. /// - public uint TrailingWord { get; private set; } + public string BrickRoot { get; private set; } = ""; + /// - /// Gets a value indicating whether valid header. + /// Gets whether a .glusterfs GFID index was present and used to locate the brick root. /// - public bool ValidHeader { get; private set; } + public bool HasGlusterIndex { get; private set; } /// - /// Initializes a new instance of . + /// Initializes a reader over one brick backing-store image. /// public GlusterFsReader(Stream stream) { - using var ms = new MemoryStream(); - stream.CopyTo(ms); - _data = ms.ToArray(); + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead) + throw new ArgumentException("GlusterFS brick image must be readable.", nameof(stream)); + + if (stream.CanSeek) { + _image = stream; + } else { + var buffered = new MemoryStream(); + stream.CopyTo(buffered); + buffered.Position = 0; + _image = buffered; + _ownsImage = true; + } + Parse(); } private void Parse() { - if (_data.Length < HeaderSize) - throw new InvalidDataException("GlusterFS: file too small for brick object header."); + if (IsXfs()) { + ParseXfs(); + return; + } + + if (IsExt()) { + ParseExt(); + return; + } + + throw new InvalidDataException( + "GlusterFS: input is not a supported brick backing-store image (expected XFS or ext2/3/4). " + + "GlusterFS itself has no standalone image magic; use a backing-filesystem image of one brick."); + } + + private bool IsXfs() { + if (_image.Length < 4) return false; + Span magic = stackalloc byte[4]; + ReadAt(0, magic); + return magic.SequenceEqual("XFSB"u8); + } + + private bool IsExt() { + if (_image.Length < ExtMagicOffset + sizeof(ushort)) return false; + Span magic = stackalloc byte[sizeof(ushort)]; + ReadAt(ExtMagicOffset, magic); + return BinaryPrimitives.ReadUInt16LittleEndian(magic) == ExtMagic; + } - if (!_data.AsSpan(0, 4).SequenceEqual(BrickMagic)) - throw new InvalidDataException("GlusterFS: missing 0xCAFE5BAB brick magic at offset 0."); + private void ParseXfs() { + _image.Position = 0; + _xfsReader = new XfsReader(_image, leaveOpen: true); + this.BackingFileSystem = "xfs"; + this.ValidHeader = true; + SetBrickRoot(_xfsReader.Entries.Select(entry => entry.Name)); + AddMetadata(_xfsReader.Entries.Count); + + foreach (var entry in _xfsReader.Entries) { + if (!TryGetBrickRelativePath(entry.Name, out var relativePath)) continue; + var captured = entry; + _entries.Add(new GlusterFsEntry { + Name = BrickPath(relativePath), + Size = entry.Size, + IsDirectory = entry.IsDirectory, + BackingPath = NormalizePath(entry.Name), + DataFactory = entry.IsDirectory ? null : () => _xfsReader!.Extract(captured), + }); + } + } - this.MagicWord = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(0, 4)); - this.TrailingWord = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(4, 4)); + private void ParseExt() { + _image.Position = 0; + _extReader = new ExtReader(_image, leaveOpen: true); + this.BackingFileSystem = "ext"; this.ValidHeader = true; + SetBrickRoot(_extReader.Entries.Select(entry => entry.Name)); + AddMetadata(_extReader.Entries.Count); - var meta = BuildMetadata(); - _entries.Add(new GlusterFsEntry { Name = "metadata.ini", Size = meta.Length, IsDirectory = false, Offset = 0, Data = meta }); - _entries.Add(new GlusterFsEntry { Name = "gluster-brick.bin", Size = _data.Length, IsDirectory = false, Offset = 0, Data = _data }); + foreach (var entry in _extReader.Entries) { + if (!TryGetBrickRelativePath(entry.Name, out var relativePath)) continue; + var captured = entry; + _entries.Add(new GlusterFsEntry { + Name = BrickPath(relativePath), + Size = entry.Size, + IsDirectory = entry.IsDirectory, + BackingPath = NormalizePath(entry.Name), + DataFactory = entry.IsDirectory ? null : () => _extReader!.Extract(captured), + }); + } } - private byte[] BuildMetadata() { - var bldr = new StringBuilder(); - bldr.Append("parse_status=detection-only\n"); - bldr.Append("format=GlusterFS (no on-disk image; brick = normal directory on local FS)\n"); - bldr.Append(CultureInfo.InvariantCulture, $"magic_word=0x{this.MagicWord:X8}\n"); - bldr.Append("magic_offset=0\n"); - bldr.Append("magic_kind=workbench-internal probe (NOT a real GlusterFS marker)\n"); - bldr.Append(CultureInfo.InvariantCulture, $"trailing_word=0x{this.TrailingWord:X8}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"image_size={_data.Length}\n"); - bldr.Append("stage=0\n"); - bldr.Append("stage_permanent=true\n"); - bldr.Append("ro_blocked_reason=GlusterFS has no on-disk image format; bricks are normal "); - bldr.Append("directories on XFS/ext4 and state lives in xattrs (trusted.gfid, "); - bldr.Append("trusted.glusterfs.*). Walking a live POSIX tree + reading xattrs is outside "); - bldr.Append("the image-stream contract.\n"); - return Encoding.UTF8.GetBytes(bldr.ToString()); + private void SetBrickRoot(IEnumerable paths) { + var roots = paths + .Select(NormalizePath) + .Where(IsGlusterIndexPath) + .Select(path => path.Equals(GlusterIndexName, StringComparison.Ordinal) + ? "" + : path[..^(GlusterIndexName.Length + 1)]) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + if (roots.Length > 1) + throw new NotSupportedException( + "GlusterFS: backing image contains multiple .glusterfs indexes; supply an image containing one brick."); + + this.HasGlusterIndex = roots.Length == 1; + this.BrickRoot = roots.FirstOrDefault() ?? ""; + } + + private bool TryGetBrickRelativePath(string path, out string relativePath) { + var normalized = NormalizePath(path); + if (this.BrickRoot.Length != 0) { + if (normalized.Equals(this.BrickRoot, StringComparison.Ordinal)) { + relativePath = ""; + return false; + } + + var prefix = this.BrickRoot + "/"; + if (!normalized.StartsWith(prefix, StringComparison.Ordinal)) { + relativePath = ""; + return false; + } + normalized = normalized[prefix.Length..]; + } + + if (normalized.Length == 0 || IsGlusterInternal(normalized)) { + relativePath = ""; + return false; + } + + relativePath = normalized; + return true; + } + + private void AddMetadata(int backingEntryCount) { + var metadata = BuildMetadata(backingEntryCount); + _entries.Add(new GlusterFsEntry { + Name = "metadata.ini", + Size = metadata.Length, + Data = metadata, + }); + } + + private byte[] BuildMetadata(int backingEntryCount) { + var builder = new StringBuilder(); + builder.Append("parse_status=single-brick-read-only\n"); + builder.Append("format=GlusterFS brick backing store\n"); + builder.Append(CultureInfo.InvariantCulture, $"backing_fs={this.BackingFileSystem}\n"); + builder.Append(CultureInfo.InvariantCulture, $"image_size={_image.Length}\n"); + builder.Append(CultureInfo.InvariantCulture, $"backing_entry_count={backingEntryCount}\n"); + builder.Append(CultureInfo.InvariantCulture, $"brick_root={this.BrickRoot}\n"); + builder.Append(CultureInfo.InvariantCulture, $"gluster_index_detected={this.HasGlusterIndex.ToString().ToLowerInvariant()}\n"); + builder.Append("view=physical single-brick namespace\n"); + builder.Append("gluster_internal_directory=.glusterfs (hidden from normal listing)\n"); + builder.Append("xattrs=readable through native backing-filesystem accessors\n"); + builder.Append("xattr_mutation=native inline/short-form subset only; descriptor remains read-only\n"); + builder.Append("cluster_namespace_reconstruction=false\n"); + builder.Append("cluster_operations=rebalance,fix-layout,remove-brick are outside the single-image abstraction\n"); + builder.Append("mutation=false\n"); + builder.Append("mutation_blocker=maintenance must preserve every trusted.gfid/trusted.glusterfs.* storage form; external/leaf/btree xattr mutation is not complete yet.\n"); + return Encoding.UTF8.GetBytes(builder.ToString()); + } + + private static string BrickPath(string path) => "brick/" + NormalizePath(path); + + private static string NormalizePath(string path) => path.Replace('\\', '/').Trim('/'); + + private static bool IsGlusterIndexPath(string path) => + path.Equals(GlusterIndexName, StringComparison.Ordinal) || + path.EndsWith("/" + GlusterIndexName, StringComparison.Ordinal); + + private static bool IsGlusterInternal(string path) => + path.Equals(GlusterIndexName, StringComparison.Ordinal) || + path.StartsWith(GlusterIndexName + "/", StringComparison.Ordinal); + + private void ReadAt(long offset, Span destination) { + var original = _image.Position; + try { + _image.Position = offset; + _image.ReadExactly(destination); + } finally { + _image.Position = original; + } } /// - /// Decodes the supplied input. + /// Extracts one surfaced entry. /// public byte[] Extract(GlusterFsEntry entry) { ArgumentNullException.ThrowIfNull(entry); - return entry.Data; + if (entry.IsDirectory) return []; + return entry.DataFactory?.Invoke() ?? entry.Data; + } + + /// + /// Reads the native extended attributes belonging to a surfaced brick entry. + /// + public IReadOnlyDictionary ReadExtendedAttributes(GlusterFsEntry entry) { + ArgumentNullException.ThrowIfNull(entry); + if (entry.BackingPath is null) + return new Dictionary(StringComparer.Ordinal); + + _image.Position = 0; + return this.BackingFileSystem switch { + "xfs" => XfsExtendedAttributes.Read(_image, entry.BackingPath), + "ext" => ExtExtendedAttributes.Read(_image, entry.BackingPath), + _ => throw new NotSupportedException($"GlusterFS: xattrs are not supported for backing filesystem '{this.BackingFileSystem}'."), + }; } /// - /// Releases resources held by this instance. + /// Releases backing filesystem readers and any spool created for a non-seekable input. /// - public void Dispose() { } + public void Dispose() { + _xfsReader?.Dispose(); + _extReader?.Dispose(); + if (_ownsImage) _image.Dispose(); + } } diff --git a/FileSystems/FileSystem.Lustre/LustreFormatDescriptor.cs b/FileSystems/FileSystem.Lustre/LustreFormatDescriptor.cs index cf9221bf6..2a65e6087 100644 --- a/FileSystems/FileSystem.Lustre/LustreFormatDescriptor.cs +++ b/FileSystems/FileSystem.Lustre/LustreFormatDescriptor.cs @@ -6,27 +6,38 @@ namespace FileSystem.Lustre; /// -/// R/O descriptor for Lustre MDT/OST images via ldiskfs (ext4-compatible) -/// reader delegation. Surfaces the ldiskfs view of a single MDT or OST -/// backing store — NOT the Lustre logical view (which would require combining -/// MDT inode metadata with file data striped across multiple OSTs, out of -/// scope without live cluster metadata). +/// Descriptor for Lustre MDT/OST images backed by ldiskfs (ext4-compatible). +/// The archive projection surfaces one target's backing-store namespace — NOT the +/// distributed Lustre logical view, which requires correlating MDT metadata with +/// objects striped across multiple OSTs. +/// +/// +/// Maintenance is intentionally narrower than generic ext editing. Free-space wipe +/// follows the ldiskfs block allocation bitmap, and shrink updates ext geometry in +/// place, preserving every surviving allocated block. Add/replace/remove, defrag and +/// structural relayout are not advertised because rebuilding through the generic ext +/// writer would discard Lustre LMA/LOV/FID extended attributes and target metadata. +/// Legacy "LUSTRE" / "LUst" object-header dumps remain inspection-only. +/// /// /// Detection is extension-routed (.lustre / .ost / .mdt) and the legacy /// "LUSTRE" / "LUst" object-header magic at offset 0; ext4 superblock magic is -/// deliberately NOT registered here (it would steal detection from generic ext4 -/// images). When opened with an ldiskfs MDT/OST image (recognised by the .ost / -/// .mdt / .lustre extension), delegates the file walk -/// to FileSystem.Ext.ExtReader. +/// deliberately NOT registered here because that would steal generic ext4 images. /// /// References: /// -/// https://www.lustre.org/ — project home -/// https://wiki.lustre.org/ — Lustre wiki (architecture, ldiskfs/MDT/OST layout) -/// https://en.wikipedia.org/wiki/Lustre_(file_system) — Wikipedia article +/// https://wiki.lustre.org/Configuring_the_Lustre_File_System — mkfs.lustre examples showing ldiskfs backing targets +/// https://wiki.lustre.org/Understanding_Lustre_Internals — backing ldiskfs inspection and Lustre target internals +/// https://docs.kernel.org/filesystems/ext4/bitmaps.html — ext4 allocation bitmap semantics and BLOCK_UNINIT warning +/// https://docs.kernel.org/filesystems/ext4/group_descr.html — group descriptor and bitmap locations /// /// -public sealed class LustreFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { +public sealed class LustreFormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IWipeEmpty, + IArchiveShrinkable, + ILayoutOptimizable { /// /// Gets the id. @@ -86,12 +97,14 @@ public sealed class LustreFormatDescriptor : IFormatDescriptor, IArchiveFormatOp /// Gets the description. /// public string Description => - "Lustre R/O via ldiskfs (ext4) reader delegation. Surfaces the ldiskfs view of one " + - "MDT or OST backing store (file walk over the ext4-compatible block layout); Lustre " + - "xattrs (LMA, LOV EA striping, FID) are preserved in the raw image but not interpreted. " + - "The Lustre logical view (combining MDT inode metadata with file data striped across " + - "multiple OSTs) requires live cluster metadata and is out of scope. Legacy 'LUSTRE'/'LUst' " + - "object-header dumps still surface as raw bytes + metadata.ini."; + "Lustre target inspection via ldiskfs (ext4) reader delegation, with conservative " + + "offline maintenance on genuine ldiskfs images: bitmap-proven free-block wipe and " + + "in-place trailing-free shrink/compact. Lustre xattrs (LMA, LOV EA striping, FID) " + + "remain opaque and are preserved because maintenance never reconstructs live inodes. " + + "Add/remove/purge, defrag and block-size relayout are intentionally not claimed: a " + + "generic ext rebuild would lose Lustre target semantics. The distributed logical view " + + "still requires live cluster metadata and is out of scope. Legacy 'LUSTRE'/'LUst' dumps " + + "remain raw inspection-only objects."; /// /// Lists the entries in the supplied container. @@ -123,4 +136,26 @@ Stream IArchiveFormatOperations.OpenEntry(Stream archive, string entryName, stri var data = r.Extract(entry); return new BoundedEntryStream(new MemoryStream(data, writable: false), data.Length, leaveOpen: false); } + + /// + /// Zeroes complete ldiskfs blocks whose initialized allocation bitmap proves them free. + /// Cluster-tip and deleted-dirent requests are deliberately ignored: those byte ranges + /// cannot be proven dead without interpreting Lustre-specific inode/xattr semantics. + /// + public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) + => LustreLdiskfsMaintenance.WipeFreeBlocks(image); + + /// + /// Shrinks an offline ldiskfs target by trimming trailing free blocks while preserving + /// all surviving allocated blocks and Lustre-specific inode/xattr bytes. + /// + public void Shrink(Stream input, Stream output) + => LustreLdiskfsMaintenance.Shrink(input, output); + + /// + /// Reports the current ldiskfs block geometry. Structural relayout is deliberately not + /// offered until a Lustre-aware writer can preserve all target metadata. + /// + public LayoutAnalysis AnalyzeLayout(Stream image) + => LustreLdiskfsMaintenance.AnalyzeLayout(image); } diff --git a/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs b/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs new file mode 100644 index 000000000..a9964adb1 --- /dev/null +++ b/FileSystems/FileSystem.Lustre/LustreLdiskfsMaintenance.cs @@ -0,0 +1,379 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using Compression.Registry; +using FileSystem.Ext; + +namespace FileSystem.Lustre; + +/// +/// Conservative maintenance primitives for an offline Lustre ldiskfs MDT/OST backing image. +/// They deliberately operate below the Lustre namespace layer: no inode, directory or Lustre +/// xattr is reconstructed. Only allocation-bitmap-proven free blocks are wiped, and shrinking +/// uses the ext in-place geometry editor so every surviving allocated block remains intact. +/// +/// +/// The layout facts used here come from the Linux ext4 on-disk documentation. In particular, +/// block-group descriptors locate the block bitmaps, one clear bitmap bit means one free block, +/// and a BLOCK_UNINIT group does not carry a trustworthy on-disk bitmap. Destructive maintenance +/// additionally requires metadata_csum and verifies the superblock, group descriptor and block +/// bitmap checksums before trusting allocation state. Lustre documents ldiskfs as its ext4-derived +/// backing filesystem. No external implementation code is copied. +/// +internal static class LustreLdiskfsMaintenance { + private const int SuperblockOffset = 1024; + private const ushort ExtMagic = 0xEF53; + + private const uint CompatSparseSuper2 = 0x0200; + + private const uint IncompatRecover = 0x0004; + private const uint IncompatJournalDev = 0x0008; + private const uint IncompatMetaBg = 0x0010; + private const uint Incompat64Bit = 0x0080; + private const uint IncompatCsumSeed = 0x2000; + + private const uint RoCompatBigalloc = 0x0200; + private const uint RoCompatMetadataCsum = 0x0400; + private const uint RoCompatReadOnly = 0x1000; + + private const ushort BgBlockUninit = 0x0002; + + private readonly record struct Geometry( + int BlockSize, + ulong TotalBlocks, + uint FirstDataBlock, + uint BlocksPerGroup, + uint GroupCount, + int DescriptorSize, + long DescriptorTableOffset, + uint FeatureCompat, + uint FeatureIncompat, + uint FeatureRoCompat, + uint ChecksumSeed); + + /// + /// Wipes only complete blocks that the initialized ldiskfs allocation bitmap proves free. + /// BLOCK_UNINIT groups are skipped rather than inferred, because ext4 explicitly permits + /// their on-disk bitmaps to contain meaningless data while fixed metadata is still present. + /// + public static long WipeFreeBlocks(Stream image) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("Lustre wipe requires a readable, writable, seekable stream.", nameof(image)); + + var geometry = ReadGeometry(image); + ValidateWritableProfile(geometry); + + var descriptor = new byte[geometry.DescriptorSize]; + var bitmap = new byte[geometry.BlockSize]; + var zeros = new byte[Math.Min(64 * 1024, Math.Max(geometry.BlockSize, 4096))]; + long wiped = 0; + + for (uint group = 0; group < geometry.GroupCount; ++group) { + ReadDescriptor(image, geometry, group, descriptor); + + // BLOCK_UNINIT is itself a safe stop condition: if corruption merely sets this + // bit we skip a writable group rather than touching it. If corruption clears the + // bit on a genuinely uninitialised group, the descriptor checksum below catches it. + var flags = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.AsSpan(0x12, 2)); + if ((flags & BgBlockUninit) != 0) + continue; + + ValidateGroupDescriptorChecksum(geometry, group, descriptor); + ReadAndValidateBlockBitmap(image, geometry, group, descriptor, bitmap); + + var groupFirst = (ulong)geometry.FirstDataBlock + (ulong)group * geometry.BlocksPerGroup; + if (groupFirst >= geometry.TotalBlocks) + break; + var blocksInGroup = (int)Math.Min((ulong)geometry.BlocksPerGroup, geometry.TotalBlocks - groupFirst); + + var runStart = -1; + for (var bit = 0; bit <= blocksInGroup; ++bit) { + var isFree = bit < blocksInGroup && (bitmap[bit >> 3] & (1 << (bit & 7))) == 0; + if (isFree) { + if (runStart < 0) runStart = bit; + continue; + } + + if (runStart < 0) + continue; + + var runBlocks = bit - runStart; + var firstBlock = groupFirst + (uint)runStart; + var offset = CheckedByteOffset(firstBlock, geometry.BlockSize); + var length = checked((long)runBlocks * geometry.BlockSize); + if (offset + length > image.Length) + throw new InvalidDataException($"Lustre/ldiskfs: free run in group {group} extends beyond the image."); + + ZeroRange(image, offset, length, zeros); + wiped = checked(wiped + length); + runStart = -1; + } + } + + image.Flush(); + return wiped; + } + + /// + /// Copies to and trims trailing + /// bitmap-free ldiskfs blocks using the existing ext in-place shrinker. There is no + /// extract/re-create fallback: such a fallback would lose Lustre-specific xattrs and + /// target metadata even when ordinary file contents happened to round-trip. + /// + public static void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("Lustre shrink requires a readable, seekable input stream.", nameof(input)); + if (!output.CanRead || !output.CanWrite || !output.CanSeek) + throw new ArgumentException("Lustre shrink requires a readable, writable, seekable output stream.", nameof(output)); + if (ReferenceEquals(input, output)) + throw new ArgumentException("Lustre shrink requires a distinct output stream so failure cannot partially rewrite the source.", nameof(output)); + + var geometry = ReadGeometry(input); + ValidateWritableProfile(geometry); + ValidateShrinkProfile(input, geometry); + + input.Position = 0; + output.Position = 0; + output.SetLength(0); + input.CopyTo(output); + output.Position = 0; + + // This path updates the ext block bitmap, group descriptors, primary + backup + // superblocks and supported checksums in place. It never re-authors the inode tree. + _ = ExtInPlaceShrinker.ShrinkToFit(output); + output.Position = 0; + } + + /// + /// Reports the allocation unit that is safe to observe without pretending that a + /// block-size conversion is currently available for Lustre target metadata. + /// + public static LayoutAnalysis AnalyzeLayout(Stream image) { + ArgumentNullException.ThrowIfNull(image); + var geometry = ReadGeometry(image); + return new LayoutAnalysis { + ImageSize = image.Length, + CurrentUnitSize = geometry.BlockSize, + CurrentSlackBytes = 0, + OptimalUnitSize = geometry.BlockSize, + OptimalSlackBytes = 0, + RequiresRebuild = [], + Notes = [ + "Lustre ldiskfs block geometry can be inspected safely in place.", + "Block-size relayout is deliberately not offered: rebuilding as generic ext would discard Lustre LMA/LOV/FID xattrs and target metadata.", + ], + }; + } + + private static Geometry ReadGeometry(Stream image) { + if (!image.CanRead || !image.CanSeek) + throw new ArgumentException("Lustre ldiskfs maintenance requires a readable, seekable stream.", nameof(image)); + if (image.Length < SuperblockOffset + 1024) + throw new InvalidDataException("Lustre/ldiskfs: image is too small for an ext superblock."); + + var superblock = new byte[1024]; + image.Position = SuperblockOffset; + image.ReadExactly(superblock); + + var magic = BinaryPrimitives.ReadUInt16LittleEndian(superblock.AsSpan(56, 2)); + if (magic != ExtMagic) + throw new InvalidDataException($"Lustre maintenance requires an ldiskfs backing image (ext magic 0xEF53, found 0x{magic:X4})."); + + var logBlockSize = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(24, 4)); + if (logBlockSize > 6) + throw new InvalidDataException($"Lustre/ldiskfs: unsupported block-size exponent {logBlockSize}."); + var blockSize = checked(1024 << (int)logBlockSize); + + var firstDataBlock = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(20, 4)); + var blocksPerGroup = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(32, 4)); + if (blocksPerGroup == 0 || (ulong)blocksPerGroup > (ulong)blockSize * 8) + throw new InvalidDataException($"Lustre/ldiskfs: invalid s_blocks_per_group value {blocksPerGroup} for {blockSize}-byte blocks."); + + var featureCompat = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(92, 4)); + var featureIncompat = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(96, 4)); + var featureRoCompat = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(100, 4)); + + uint checksumSeed = 0; + if ((featureRoCompat & RoCompatMetadataCsum) != 0) { + ValidateSuperblockChecksum(superblock); + checksumSeed = (featureIncompat & IncompatCsumSeed) != 0 + ? BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(0x270, 4)) + : Crc32c(0xFFFFFFFFu, superblock.AsSpan(104, 16)); + } + + ulong totalBlocks = BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(4, 4)); + if ((featureIncompat & Incompat64Bit) != 0) + totalBlocks |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(superblock.AsSpan(0x150, 4)) << 32; + if (totalBlocks <= firstDataBlock) + throw new InvalidDataException("Lustre/ldiskfs: invalid block count."); + + var descriptorSize = 32; + if ((featureIncompat & Incompat64Bit) != 0) { + descriptorSize = BinaryPrimitives.ReadUInt16LittleEndian(superblock.AsSpan(0xFE, 2)); + if (descriptorSize < 64 || descriptorSize > blockSize || (descriptorSize & 7) != 0) + throw new InvalidDataException($"Lustre/ldiskfs: invalid 64-bit group descriptor size {descriptorSize}."); + } + + var groupCount64 = (totalBlocks - firstDataBlock + blocksPerGroup - 1) / blocksPerGroup; + if (groupCount64 is 0 or > uint.MaxValue) + throw new InvalidDataException($"Lustre/ldiskfs: unsupported block-group count {groupCount64}."); + var groupCount = (uint)groupCount64; + + var descriptorTableOffset = checked((long)(firstDataBlock + 1UL) * blockSize); + var descriptorBytes = checked((long)groupCount * descriptorSize); + if (descriptorTableOffset < 0 || descriptorTableOffset + descriptorBytes > image.Length) + throw new InvalidDataException("Lustre/ldiskfs: primary group descriptor table is truncated."); + + if (totalBlocks > (ulong)(long.MaxValue / blockSize)) + throw new InvalidDataException("Lustre/ldiskfs: volume is too large for stream byte addressing."); + var declaredBytes = checked((long)totalBlocks * blockSize); + if (declaredBytes > image.Length) + throw new InvalidDataException($"Lustre/ldiskfs: superblock declares {declaredBytes} bytes but the image contains only {image.Length}."); + + return new Geometry( + blockSize, totalBlocks, firstDataBlock, blocksPerGroup, groupCount, + descriptorSize, descriptorTableOffset, featureCompat, featureIncompat, featureRoCompat, + checksumSeed); + } + + private static void ValidateWritableProfile(Geometry geometry) { + if ((geometry.FeatureRoCompat & RoCompatMetadataCsum) == 0) + throw new NotSupportedException( + "Lustre destructive maintenance requires ldiskfs metadata_csum so corrupted allocation bitmaps can be detected before any block is overwritten."); + if ((geometry.FeatureIncompat & IncompatRecover) != 0) + throw new NotSupportedException("Lustre maintenance refuses an ldiskfs image that still needs journal recovery; replay it first."); + if ((geometry.FeatureIncompat & IncompatJournalDev) != 0) + throw new NotSupportedException("Lustre maintenance does not support ldiskfs images with an external journal device."); + if ((geometry.FeatureIncompat & IncompatMetaBg) != 0) + throw new NotSupportedException("Lustre maintenance does not yet support ext META_BG descriptor placement."); + if ((geometry.FeatureRoCompat & RoCompatBigalloc) != 0) + throw new NotSupportedException("Lustre maintenance does not yet support ext BIGALLOC cluster bitmaps."); + if ((geometry.FeatureRoCompat & RoCompatReadOnly) != 0) + throw new NotSupportedException("Lustre maintenance refuses an ext image explicitly marked read-only."); + } + + private static void ValidateShrinkProfile(Stream image, Geometry geometry) { + if (geometry.TotalBlocks > uint.MaxValue) + throw new NotSupportedException("Lustre shrink currently supports ldiskfs volumes up to UInt32 block counts."); + if ((geometry.FeatureCompat & CompatSparseSuper2) != 0) + throw new NotSupportedException("Lustre shrink does not yet rewrite SPARSE_SUPER2 backup locations."); + + var descriptor = new byte[geometry.DescriptorSize]; + var bitmap = new byte[geometry.BlockSize]; + for (uint group = 0; group < geometry.GroupCount; ++group) { + ReadDescriptor(image, geometry, group, descriptor); + var flags = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.AsSpan(0x12, 2)); + if ((flags & BgBlockUninit) != 0) + throw new NotSupportedException( + $"Lustre shrink refuses lazy BLOCK_UNINIT group {group}; its on-disk allocation bitmap is not authoritative yet."); + + ValidateGroupDescriptorChecksum(geometry, group, descriptor); + ReadAndValidateBlockBitmap(image, geometry, group, descriptor, bitmap); + } + } + + private static void ReadDescriptor(Stream image, Geometry geometry, uint group, byte[] destination) { + var offset = checked(geometry.DescriptorTableOffset + (long)group * geometry.DescriptorSize); + if (offset + geometry.DescriptorSize > image.Length) + throw new InvalidDataException($"Lustre/ldiskfs: group descriptor {group} is truncated."); + image.Position = offset; + image.ReadExactly(destination); + } + + private static void ReadAndValidateBlockBitmap( + Stream image, + Geometry geometry, + uint group, + ReadOnlySpan descriptor, + byte[] bitmap) { + var bitmapBlock = ReadBlockBitmapBlock(descriptor, geometry.DescriptorSize); + if (bitmapBlock >= geometry.TotalBlocks) + throw new InvalidDataException($"Lustre/ldiskfs: group {group} block bitmap points outside the volume ({bitmapBlock})."); + + var bitmapOffset = CheckedByteOffset(bitmapBlock, geometry.BlockSize); + if (bitmapOffset + geometry.BlockSize > image.Length) + throw new InvalidDataException($"Lustre/ldiskfs: group {group} block bitmap is truncated."); + + image.Position = bitmapOffset; + image.ReadExactly(bitmap); + ValidateBlockBitmapChecksum(geometry, group, descriptor, bitmap); + } + + private static ulong ReadBlockBitmapBlock(ReadOnlySpan descriptor, int descriptorSize) { + ulong block = BinaryPrimitives.ReadUInt32LittleEndian(descriptor[..4]); + if (descriptorSize >= 64) + block |= (ulong)BinaryPrimitives.ReadUInt32LittleEndian(descriptor.Slice(0x20, 4)) << 32; + return block; + } + + private static void ValidateSuperblockChecksum(ReadOnlySpan superblock) { + var expected = BinaryPrimitives.ReadUInt32LittleEndian(superblock.Slice(0x3FC, 4)); + var actual = Crc32c(0xFFFFFFFFu, superblock[..0x3FC]); + if (actual != expected) + throw new InvalidDataException( + $"Lustre/ldiskfs: superblock metadata checksum mismatch (stored 0x{expected:X8}, calculated 0x{actual:X8})."); + } + + private static void ValidateGroupDescriptorChecksum(Geometry geometry, uint group, byte[] descriptor) { + var expected = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.AsSpan(0x1E, 2)); + var copy = (byte[])descriptor.Clone(); + BinaryPrimitives.WriteUInt16LittleEndian(copy.AsSpan(0x1E, 2), 0); + + Span groupBytes = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(groupBytes, group); + var actual = Crc32c(Crc32c(geometry.ChecksumSeed, groupBytes), copy); + if ((ushort)actual != expected) + throw new InvalidDataException( + $"Lustre/ldiskfs: group {group} descriptor checksum mismatch (stored 0x{expected:X4}, calculated 0x{(ushort)actual:X4})."); + } + + private static void ValidateBlockBitmapChecksum( + Geometry geometry, + uint group, + ReadOnlySpan descriptor, + ReadOnlySpan bitmap) { + var meaningfulBytes = checked((int)(((ulong)geometry.BlocksPerGroup + 7) / 8)); + if (meaningfulBytes > bitmap.Length) + throw new InvalidDataException($"Lustre/ldiskfs: group {group} block bitmap geometry exceeds its bitmap block."); + + var actual = Crc32c(geometry.ChecksumSeed, bitmap[..meaningfulBytes]); + uint expected = BinaryPrimitives.ReadUInt16LittleEndian(descriptor.Slice(0x18, 2)); + if (geometry.DescriptorSize >= 64) + expected |= (uint)BinaryPrimitives.ReadUInt16LittleEndian(descriptor.Slice(0x38, 2)) << 16; + + var matches = geometry.DescriptorSize >= 64 + ? actual == expected + : (ushort)actual == (ushort)expected; + if (!matches) + throw new InvalidDataException( + $"Lustre/ldiskfs: group {group} block bitmap checksum mismatch (stored 0x{expected:X8}, calculated 0x{actual:X8})."); + } + + private static uint Crc32c(uint seed, ReadOnlySpan data) { + const uint polynomial = 0x82F63B78u; + var crc = seed; + foreach (var value in data) { + crc ^= value; + for (var bit = 0; bit < 8; ++bit) + crc = (crc & 1) != 0 ? (crc >> 1) ^ polynomial : crc >> 1; + } + return crc; + } + + private static long CheckedByteOffset(ulong block, int blockSize) { + if (block > (ulong)(long.MaxValue / blockSize)) + throw new InvalidDataException("Lustre/ldiskfs: block address exceeds stream addressing range."); + return checked((long)block * blockSize); + } + + private static void ZeroRange(Stream image, long offset, long length, byte[] zeros) { + image.Position = offset; + while (length > 0) { + var chunk = (int)Math.Min(length, zeros.Length); + image.Write(zeros, 0, chunk); + length -= chunk; + } + } +} diff --git a/FileSystems/FileSystem.Lustre/LustreReader.cs b/FileSystems/FileSystem.Lustre/LustreReader.cs index fb5b13e14..5b5651f75 100644 --- a/FileSystems/FileSystem.Lustre/LustreReader.cs +++ b/FileSystems/FileSystem.Lustre/LustreReader.cs @@ -193,9 +193,10 @@ private byte[] BuildLdiskfsMetadata() { bldr.Append(CultureInfo.InvariantCulture, $"image_size={_data.Length}\n"); bldr.Append(CultureInfo.InvariantCulture, $"ldiskfs_volume_label={this.LdiskfsVolumeLabel}\n"); bldr.Append(CultureInfo.InvariantCulture, $"ldiskfs_entry_count={(_extReader?.Entries.Count ?? 0)}\n"); - bldr.Append("note=R/O via ext4 reader delegation. Surfaces the ldiskfs view of a single MDT or OST "); - bldr.Append("backing store, NOT the Lustre logical view. Lustre xattrs (LMA, LOV EA striping, FID) "); - bldr.Append("are preserved in the raw image but not interpreted. Combining MDT inode metadata with "); + bldr.Append("note=Read projection via ext4 delegation. Conservative offline maintenance may wipe bitmap-proven "); + bldr.Append("free blocks and trim trailing free ldiskfs blocks without reconstructing live inodes. The view is one "); + bldr.Append("MDT or OST backing store, NOT the Lustre logical view. Lustre xattrs (LMA, LOV EA striping, FID) "); + bldr.Append("remain opaque and allocated metadata is preserved byte-for-byte. Combining MDT inode metadata with "); bldr.Append("file data striped across multiple OSTs requires live cluster metadata and is out-of-scope.\n"); return Encoding.UTF8.GetBytes(bldr.ToString()); } diff --git a/FileSystems/FileSystem.MooseFs/MooseFsFormatDescriptor.cs b/FileSystems/FileSystem.MooseFs/MooseFsFormatDescriptor.cs index cf1155581..3a0086381 100644 --- a/FileSystems/FileSystem.MooseFs/MooseFsFormatDescriptor.cs +++ b/FileSystems/FileSystem.MooseFs/MooseFsFormatDescriptor.cs @@ -6,115 +6,171 @@ namespace FileSystem.MooseFs; /// -/// Partial R/O descriptor for MooseFS master-metadata images -/// (metadata.mfs). Surfaces the metadata envelope (signature, -/// counters, section index) and the raw payload bytes of each walked -/// section. Path-tree (NODE/EDGE) and chunk-id (CHNK) bodies are -/// version-specific and require golden samples to decode honestly — the -/// reader makes no claim about their internal structure. +/// Descriptor for MooseFS master-metadata images (metadata.mfs). +/// The standalone image contains namespace/chunk metadata, not file payloads; +/// payload bytes live on chunk servers. The descriptor therefore exposes a +/// synthetic forensic view of the metadata envelope rather than pretending a +/// metadata dump is a self-contained MooseFS volume. /// /// -/// MooseFS file content lives on chunk servers and is unreachable from a -/// single metadata image. Listing therefore surfaces ONLY synthetic -/// metadata + per-section raw payloads, never POSIX paths. +/// Safe standalone-image maintenance is deliberately narrow: byte layout can +/// be mapped, unused-space wiping fails closed (valid metadata dumps are tightly +/// packed), and purge resets the image to MooseFS's official MFSM NEW +/// empty bootstrap. Add/replace/remove, shrink and defragmentation require +/// semantic NODE/EDGE/CHNK updates and/or live cluster coordination and are not +/// advertised here. /// /// /// References: /// -/// https://github.com/moosefs/moosefs — canonical source (master metadata dump/load code) -/// https://moosefs.com/ — vendor site and documentation -/// https://en.wikipedia.org/wiki/Moose_File_System — Wikipedia article +/// https://github.com/moosefs/moosefs — canonical implementation and metadata loader/store format +/// https://moosefs.com/ — vendor documentation /// /// -public sealed class MooseFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { +public sealed class MooseFsFormatDescriptor + : IFormatDescriptor, IArchiveFormatOperations, IArchiveLayoutMap, IArchivePurgeable { - /// - /// Gets the id. - /// + /// Gets the id. public string Id => "MooseFs"; - /// - /// Gets the display name. - /// + + /// Gets the display name. public string DisplayName => "MooseFS"; - /// - /// Gets the category. - /// + + /// Gets the category. public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// + + /// Gets the capabilities. public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest; - /// - /// Gets the default extension. - /// + + /// Gets the default extension. public string DefaultExtension => ".mfsm"; - // Note: ".mfs" intentionally NOT claimed — collides with Macintosh File System (FileSystem.Mfs). - // MooseFS detection is by magic 'MFSM' at offset 0; the ".mfsm" extension is MooseFS-specific. - /// - /// Gets the extensions. - /// + + // ".mfs" intentionally is not claimed: it collides with Macintosh File System (FileSystem.Mfs). + // MooseFS detection is by the MFSM signature at offset zero. + /// Gets the extensions. public IReadOnlyList Extensions => [".mfsm"]; - /// - /// Gets the compound extensions. - /// + + /// Gets the compound extensions. public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// + + /// Gets the magic signatures. public IReadOnlyList MagicSignatures => [ - // ASCII "MFSM" at offset 0 — MooseFS master metadata tag. new("MFSM"u8.ToArray(), Offset: 0, Confidence: 0.90), ]; - /// - /// Gets the methods. - /// + + /// Gets the methods. public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// + + /// Gets the tar compression format id. public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// + + /// Gets the family. public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// + + /// Gets the description. public string Description => - "MooseFS — partial R/O of master metadata envelope (signature + section " + - "index). File content lives on chunk servers and is unreachable from a " + - "single metadata image; only metadata.ini + raw image + per-section " + - "raw payloads are surfaced."; + "MooseFS master metadata image. Reads the versioned metadata envelope and section table; " + + "file payloads remain on chunk servers. Standalone maintenance provides byte-layout mapping, " + + "safe unused-space wiping and destructive reset to the official empty MFSM NEW bootstrap; " + + "full namespace R/W, shrink and defragmentation are intentionally not claimed."; + + /// Lists the synthetic entries in the supplied metadata image. + public List List(Stream stream, string? password) { + using var reader = new MooseFsReader(stream); + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo( + index, entry.Name, entry.Size, entry.Size, "Stored", entry.IsDirectory, false, null)).ToList(); + } + + /// Extracts the synthetic metadata-image views. + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + using var reader = new MooseFsReader(stream); + foreach (var entry in reader.Entries) { + if (entry.IsDirectory) + continue; + if (files != null && !MatchesFilter(entry.Name, files)) + continue; + WriteFile(outputDir, entry.Name, reader.Extract(entry)); + } + } /// - /// Lists the entries in the supplied container. + /// Enumerates the exact bytes occupied by a valid MooseFS metadata image. + /// Unknown/truncated images return no extents so generic maintenance fails closed. /// - public List List(Stream stream, string? password) { - var r = new MooseFsReader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo( - i, e.Name, e.Size, e.Size, "Stored", e.IsDirectory, false, null)).ToList(); + public IEnumerable EnumerateLayout(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanSeek) + return []; + + var originalPosition = archive.Position; + try { + archive.Position = 0; + using var reader = new MooseFsReader(archive); + if (reader.ParseStatus != "ok") + return []; + + if (reader.IsEmptyBootstrap) + return [new DefragBlockInfo(0, reader.ImageSize, DefragBlockKind.MetadataReserved, "MFSM NEW bootstrap")]; + + if (reader.FileFormatVersion is byte version && version < 0x16) + return [new DefragBlockInfo(0, reader.ImageSize, DefragBlockKind.MetadataReserved, "legacy MooseFS metadata")]; + + var result = new List { + new(0, 24, DefragBlockKind.MetadataReserved, "MooseFS metadata header"), + }; + + foreach (var section in reader.Sections) { + result.Add(new DefragBlockInfo( + section.Offset - 16, 16, DefragBlockKind.MetadataReserved, $"{section.Tag} section header")); + if (section.Length > 0) + result.Add(new DefragBlockInfo(section.Offset, section.Length, DefragBlockKind.Used, section.Tag)); + } + + result.Add(new DefragBlockInfo( + reader.ImageSize - MooseFsReader.EofMarker.Length, + MooseFsReader.EofMarker.Length, + DefragBlockKind.MetadataReserved, + "MooseFS EOF marker")); + return result; + } catch (InvalidDataException) { + return []; + } finally { + archive.Position = originalPosition; + } } /// - /// Decodes the supplied input. + /// Resets a valid metadata image to MooseFS's official empty bootstrap state. + /// This only rewrites metadata.mfs; any chunk-server data is outside the + /// image and therefore outside this operation. /// - public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new MooseFsReader(stream); - foreach (var e in r.Entries) { - if (e.IsDirectory) continue; - if (files != null && !MatchesFilter(e.Name, files)) continue; - WriteFile(outputDir, e.Name, r.Extract(e)); + public void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException( + "MooseFS purge requires a readable, writable, seekable stream.", nameof(archive)); + + archive.Position = 0; + using (var reader = new MooseFsReader(archive)) { + if (reader.ParseStatus != "ok") + throw new InvalidDataException( + $"MooseFS purge requires a structurally valid metadata image; parse status is '{reader.ParseStatus}'."); } + + archive.Position = 0; + archive.Write("MFSM NEW"u8); + archive.SetLength(8); + archive.Flush(); } Stream IArchiveFormatOperations.OpenEntry(Stream archive, string entryName, string? password) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(entryName); - var r = new MooseFsReader(archive); - var entry = r.Entries.FirstOrDefault(e => e.Name == entryName) + using var reader = new MooseFsReader(archive); + var entry = reader.Entries.FirstOrDefault(e => e.Name == entryName) ?? throw new FileNotFoundException($"MooseFS entry not found: {entryName}"); - var data = r.Extract(entry); + var data = reader.Extract(entry); return new BoundedEntryStream(new MemoryStream(data, writable: false), data.Length, leaveOpen: false); } } diff --git a/FileSystems/FileSystem.MooseFs/MooseFsReader.cs b/FileSystems/FileSystem.MooseFs/MooseFsReader.cs index 14245eebc..b9f5ca915 100644 --- a/FileSystems/FileSystem.MooseFs/MooseFsReader.cs +++ b/FileSystems/FileSystem.MooseFs/MooseFsReader.cs @@ -6,110 +6,89 @@ namespace FileSystem.MooseFs; /// -/// Partial R/O reader for MooseFS master-metadata images (metadata.mfs). -/// MooseFS is a fault-tolerant distributed FS — the master server keeps the -/// namespace + chunk-server topology in a single binary metadata file, while -/// file data lives on chunk servers. This reader understands the master -/// metadata's outer envelope: -/// -/// 8-byte ASCII signature (e.g. MFSM 2.0, MFSM 1.6, -/// MFSM 1.5, MFSM 1.4, MFSM NEW). -/// For 1.6+ images: two 8-byte big-endian counters -/// (file-id counter, metadata version) immediately after the signature. -/// Sequence of sections. Each section: 8-byte ASCII type tag -/// (SESS 1.0, STAT 1.0, NODE 1.0, EDGE 1.0, -/// FREE 1.0, XATR 1.0, CHNK 1.0, OPEN 1.0, -/// FLCK 1.0, QUOT 1.0, ACLS 1.0, …) + 8-byte -/// big-endian payload length + that many payload bytes. -/// Final 16-byte terminator [MFS EOF MARKER]. -/// -/// -/// -/// The reader walks the section index only — it does not attempt to -/// decode NODE / EDGE record bodies, which differ between MooseFS minor -/// versions and require ground-truth golden samples to validate. NODE/EDGE -/// would give path tree + inode metadata; CHNK gives chunk-id mappings. None -/// of those by themselves yield file content — MooseFS data lives on chunk -/// servers and is only reachable via the live MooseFS protocol. Therefore -/// the reader exposes: -/// -/// -/// metadata.ini — human-readable summary of header + section -/// table (name, payload offset, payload length). -/// moosefs-master.bin — the raw image, byte-for-byte. -/// section_<NAME>.bin — the raw payload bytes of each -/// section the index walk surfaced (NODE, EDGE, CHNK, …). Useful for -/// offline forensics; we make no claim about their internal -/// structure. -/// -/// -/// -/// If section-walk fails (signature past the 8-byte tag is not recognised, -/// a section length runs past EOF, the EOF marker is missing, …), the -/// reader falls back to a header-only surface (metadata.ini + raw) and -/// records the parse failure in metadata.ini's parse_status -/// field. This is the honest "we recognise the envelope but couldn't walk -/// the contents" mode rather than silently inventing entries. -/// +/// Partial reader for MooseFS master-metadata images (metadata.mfs). +/// It understands the versioned outer envelope and section framing, but does +/// not decode the version-specific NODE / EDGE / CHNK bodies or contact chunk +/// servers. Consequently all surfaced archive entries are synthetic forensic +/// views of the metadata image rather than the mounted MooseFS namespace. /// public sealed class MooseFsReader : IDisposable { - /// MooseFS master metadata 4-byte prefix: ASCII "MFSM". + /// MooseFS master metadata 4-byte prefix: ASCII MFSM. public static readonly byte[] MasterTag = "MFSM"u8.ToArray(); - /// 16-byte MooseFS end-of-file marker following the last section. + /// Modern (1.6+) 16-byte MooseFS end-of-file marker. public static readonly byte[] EofMarker = "[MFS EOF MARKER]"u8.ToArray(); - private const int HeaderSize = 8; - // After the 8-byte signature, modern (1.6+) images have two BE uint64s - // followed by the section stream. Section tag + length = 16 bytes. - private const int SectionTagSize = 8; - private const int SectionLenSize = 8; - // Section payloads can be huge on a real cluster — cap what we'll hold in - // memory as a synthetic per-section entry. Larger sections still show up - // in metadata.ini's table but their raw payload is not surfaced. + private const int SignatureSize = 8; + private const int MetadataHeaderSize = 16; + private const int SectionHeaderSize = 16; private const long MaxInMemorySection = 64L * 1024 * 1024; private readonly byte[] _data; private readonly List _entries = []; private readonly List _sections = []; - /// Listing of every entry this image surfaces. + /// Listing of every synthetic entry surfaced from this image. public IReadOnlyList Entries => _entries; - /// Section index walked from the master metadata stream. + /// Section index walked from section-framed (1.6+) metadata. public IReadOnlyList Sections => _sections; - /// The 8-byte ASCII signature at offset 0 (e.g. "MFSM 2.0"). + /// The 8-byte ASCII signature, for example MFSM 2.0. public string Signature { get; private set; } = ""; - /// True when the 4-byte MFSM tag was present at offset 0. + /// True once the MFSM prefix has been verified. public bool ValidHeader { get; private set; } /// - /// File-id counter from the modern (1.6+) post-signature header, or - /// null when the image is too short or pre-1.6. + /// Compatibility view of the first eight bytes after the signature, matching + /// the value older CompressionWorkbench builds exposed under this incorrect + /// name. MooseFS does not define this field as a file-id counter. /// - public ulong? FileIdCounter { get; private set; } + [Obsolete("MooseFS metadata has no file-id counter at this location; use MetadataVersion, MetaId, MaxNodeId and NextSessionId.")] + public ulong? FileIdCounter + => _data.Length >= SignatureSize + 8 + ? BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(SignatureSize, 8)) + : null; /// - /// Metadata version counter from the modern (1.6+) post-signature header, - /// or null when the image is too short or pre-1.6. + /// Packed file-format version (0x16 for 1.6, 0x20 for 2.0), + /// or null for the special MFSM NEW bootstrap image. /// + public byte? FileFormatVersion { get; private set; } + + /// Whether this is MooseFS's official eight-byte empty bootstrap image. + public bool IsEmptyBootstrap { get; private set; } + + /// + /// Maximum node id from the pre-2.0 metadata header. Not present in 2.0+. + /// + public uint? MaxNodeId { get; private set; } + + /// Metadata/changelog version stored in the master metadata header. public ulong? MetadataVersion { get; private set; } /// - /// Human-readable description of how the section walk terminated: - /// "ok" (full walk + EOF marker), "truncated" (section walk - /// stopped before EOF marker), "header-only" (image too short for - /// any sections), or "unsupported-header" (no MFSM tag). + /// Next session id from the pre-2.0 metadata header. Not present in 2.0+. /// - public string ParseStatus { get; private set; } = "unsupported-header"; + public uint? NextSessionId { get; private set; } + + /// Metadata instance id stored by 2.0+ images. + public ulong? MetaId { get; private set; } + + /// The exact image size consumed by this reader. + public long ImageSize => _data.LongLength; /// - /// Initializes a new instance of . + /// Human-readable parse result: ok, header-only, + /// truncated, trailing-data, or unsupported-header. /// + public string ParseStatus { get; private set; } = "unsupported-header"; + + /// Initializes a reader over one MooseFS metadata image. public MooseFsReader(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); using var ms = new MemoryStream(); stream.CopyTo(ms); _data = ms.ToArray(); @@ -117,145 +96,169 @@ public MooseFsReader(Stream stream) { } private void Parse() { - if (_data.Length < HeaderSize) - throw new InvalidDataException("MooseFS: file too small for master metadata header."); + if (_data.Length < SignatureSize) + throw new InvalidDataException("MooseFS: file too small for master metadata signature."); - if (!_data.AsSpan(0, 4).SequenceEqual(MasterTag)) + if (!_data.AsSpan(0, MasterTag.Length).SequenceEqual(MasterTag)) throw new InvalidDataException("MooseFS: missing 'MFSM' tag at offset 0."); this.ValidHeader = true; - this.Signature = Encoding.ASCII.GetString(_data, 0, 8).TrimEnd('\0'); - - // Modern (1.6+) images carry two BE uint64s right after the 8-byte - // signature. Older 1.4/1.5 images stream sections immediately. We pull - // the counters when the image is long enough; downstream code does not - // assume they're meaningful for non-1.6+ signatures. - if (_data.Length >= HeaderSize + 16) { - this.FileIdCounter = BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(HeaderSize, 8)); - this.MetadataVersion = BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(HeaderSize + 8, 8)); + this.Signature = Encoding.ASCII.GetString(_data, 0, SignatureSize); + + if (this.Signature == "MFSM NEW") { + this.IsEmptyBootstrap = true; + this.ParseStatus = _data.Length == SignatureSize ? "ok" : "trailing-data"; + BuildEntries(); + return; } - WalkSections(); + if (!TryParseFileFormatVersion(_data.AsSpan(0, SignatureSize), out var fileVersion)) { + this.ParseStatus = "unsupported-header"; + BuildEntries(); + return; + } - var meta = BuildMetadata(); - _entries.Add(new MooseFsEntry { - Name = "metadata.ini", - Size = meta.Length, - IsDirectory = false, - Offset = 0, - Data = meta, - }); - _entries.Add(new MooseFsEntry { - Name = "moosefs-master.bin", - Size = _data.Length, - IsDirectory = false, - Offset = 0, - Data = _data, - }); + this.FileFormatVersion = fileVersion; + if (_data.Length < SignatureSize + MetadataHeaderSize) { + this.ParseStatus = "header-only"; + BuildEntries(); + return; + } - foreach (var s in _sections) { - // Bound the per-section payload we materialise so a 4-GB CHNK section - // does not allocate 4 GB of synthetic-entry buffer. The section still - // appears in metadata.ini's table — we just refuse to mirror the - // payload as a separate entry. The full raw image stays accessible - // via moosefs-master.bin. - if (s.Length <= 0 || s.Length > MaxInMemorySection) - continue; - // Defensive: the walker already bounds Offset+Length to _data.Length, - // but re-check before slicing to guard against any future refactor. - if (s.Offset < 0 || s.Offset + s.Length > _data.Length) - continue; - var payload = _data.AsSpan((int)s.Offset, (int)s.Length).ToArray(); - _entries.Add(new MooseFsEntry { - Name = $"section_{SanitiseSectionName(s.Tag)}.bin", - Size = payload.Length, - IsDirectory = false, - Offset = s.Offset, - Data = payload, - }); + ParseMetadataHeader(fileVersion); + + if (fileVersion < 0x16) + ParseLegacyBody(); + else + WalkSections(); + + BuildEntries(); + } + + private void ParseMetadataHeader(byte fileVersion) { + var header = _data.AsSpan(SignatureSize, MetadataHeaderSize); + if (fileVersion >= 0x20) { + this.MetadataVersion = BinaryPrimitives.ReadUInt64BigEndian(header[..8]); + this.MetaId = BinaryPrimitives.ReadUInt64BigEndian(header[8..]); + return; } + + this.MaxNodeId = BinaryPrimitives.ReadUInt32BigEndian(header[..4]); + this.MetadataVersion = BinaryPrimitives.ReadUInt64BigEndian(header.Slice(4, 8)); + this.NextSessionId = BinaryPrimitives.ReadUInt32BigEndian(header[12..]); } - private void WalkSections() { - // Section stream begins after signature for 1.4/1.5, after signature + 16 - // counter bytes for 1.6+. We pick the offset by checking signature - // version: anything other than the 1.4/1.5 strings gets the modern - // offset. This still correctly rejects malformed images because the - // first section tag must be all-ASCII printable. - var sectionStart = HeaderSize; - if (this.Signature is not ("MFSM 1.4" or "MFSM 1.5") && _data.Length >= HeaderSize + 16) - sectionStart = HeaderSize + 16; - - if (_data.Length < sectionStart + SectionTagSize + SectionLenSize) { - this.ParseStatus = "header-only"; + private void ParseLegacyBody() { + // Before metadata format 1.6 MooseFS did not wrap NODE/EDGE/etc. in the + // modern 16-byte section envelope. The authoritative checker only relies + // on the final 16 zero bytes, so preserve the whole legacy body as opaque + // metadata instead of guessing record boundaries. + var minimumLength = SignatureSize + MetadataHeaderSize + 16; + if (_data.Length < minimumLength) { + this.ParseStatus = "truncated"; return; } - var offset = sectionStart; - while (offset + SectionTagSize + SectionLenSize <= _data.Length) { - // EOF marker is 16 bytes [MFS EOF MARKER] and is NOT followed by a - // length field — once we see it, the walk is complete. - if (offset + EofMarker.Length <= _data.Length - && _data.AsSpan(offset, EofMarker.Length).SequenceEqual(EofMarker)) { - this.ParseStatus = "ok"; + this.ParseStatus = _data.AsSpan(_data.Length - 16, 16).IndexOfAnyExcept((byte)0) < 0 + ? "ok" + : "truncated"; + } + + private void WalkSections() { + var offset = SignatureSize + MetadataHeaderSize; + while (offset + SectionHeaderSize <= _data.Length) { + if (_data.AsSpan(offset, EofMarker.Length).SequenceEqual(EofMarker)) { + this.ParseStatus = offset + EofMarker.Length == _data.Length ? "ok" : "trailing-data"; return; } - var tagBytes = _data.AsSpan(offset, SectionTagSize); + var tagBytes = _data.AsSpan(offset, 8); if (!IsPlausibleSectionTag(tagBytes)) { - // Section walk derailed. Keep what we have, mark truncated. We do - // not throw — header-level surfaces remain useful even if section - // walk failed (older / future MooseFS versions, partially-written - // images, dump tools that strip framing, …). this.ParseStatus = "truncated"; return; } - var tag = Encoding.ASCII.GetString(tagBytes).TrimEnd(); - var lenU64 = BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(offset + SectionTagSize, SectionLenSize)); - // A single section larger than long.MaxValue would mean the image is - // pathological or this isn't really a section header — treat as walk - // failure rather than overflowing int math. - if (lenU64 > long.MaxValue) { + + var length64 = BinaryPrimitives.ReadUInt64BigEndian(_data.AsSpan(offset + 8, 8)); + if (length64 > long.MaxValue) { this.ParseStatus = "truncated"; return; } - var len = (long)lenU64; - var payloadOffset = (long)offset + SectionTagSize + SectionLenSize; - if (payloadOffset + len > _data.Length) { - // Section claims more bytes than the image has — truncated dump or - // bogus length. Record what we got and stop. + + var length = (long)length64; + var payloadOffset = (long)offset + SectionHeaderSize; + if (length > _data.LongLength - payloadOffset) { this.ParseStatus = "truncated"; return; } - _sections.Add(new SectionEntry(tag, payloadOffset, len)); - offset = (int)(payloadOffset + len); + var tag = Encoding.ASCII.GetString(tagBytes); + _sections.Add(new SectionEntry(tag, payloadOffset, length)); + offset = checked((int)(payloadOffset + length)); } - // Walked past the last possible section header but never saw the EOF - // marker — image is truncated. The accumulated sections are still real. this.ParseStatus = "truncated"; } - private static bool IsPlausibleSectionTag(ReadOnlySpan tagBytes) { - // A valid MooseFS section tag is 8 ASCII bytes: 4-char family - // ("SESS", "NODE", "EDGE", "CHNK", "FREE", "XATR", "STAT", "OPEN", - // "FLCK", "QUOT", "ACLS", …), a space, then a version string like - // "1.0" or "2.0". We sanity-check that every byte is printable ASCII — - // this rejects random binary that just happens to be at the right - // offset without locking us to a fixed family allow-list (MooseFS - // adds new section types between minor versions). - foreach (var b in tagBytes) - if (b is < 0x20 or > 0x7E) - return false; + private void BuildEntries() { + var metadata = BuildMetadata(); + _entries.Add(new MooseFsEntry { + Name = "metadata.ini", + Size = metadata.Length, + IsDirectory = false, + Offset = 0, + Data = metadata, + }); + _entries.Add(new MooseFsEntry { + Name = "moosefs-master.bin", + Size = _data.Length, + IsDirectory = false, + Offset = 0, + Data = _data, + }); + + foreach (var section in _sections) { + if (section.Length <= 0 || section.Length > MaxInMemorySection) + continue; + if (section.Offset < 0 || section.Offset + section.Length > _data.LongLength) + continue; + + var payload = _data.AsSpan((int)section.Offset, (int)section.Length).ToArray(); + _entries.Add(new MooseFsEntry { + Name = $"section_{SanitiseSectionName(section.Tag)}.bin", + Size = payload.Length, + IsDirectory = false, + Offset = section.Offset, + Data = payload, + }); + } + } + + private static bool TryParseFileFormatVersion(ReadOnlySpan signature, out byte version) { + version = 0; + if (signature.Length != SignatureSize + || !signature[..5].SequenceEqual("MFSM "u8) + || signature[5] is < (byte)'1' or > (byte)'9' + || signature[6] != (byte)'.' + || signature[7] is < (byte)'0' or > (byte)'9') + return false; + + version = (byte)(((signature[5] - (byte)'0') << 4) | (signature[7] - (byte)'0')); return true; } + private static bool IsPlausibleSectionTag(ReadOnlySpan tag) { + if (tag.Length != 8 || tag[4] != (byte)' ' || tag[6] != (byte)'.') + return false; + + for (var i = 0; i < 4; ++i) + if (tag[i] is not (>= (byte)'A' and <= (byte)'Z') and not (>= (byte)'0' and <= (byte)'9')) + return false; + + return tag[5] is >= (byte)'0' and <= (byte)'9' + && tag[7] is >= (byte)'0' and <= (byte)'9'; + } + private static string SanitiseSectionName(string tag) { - // Section tags include a space ("NODE 1.0") which makes for awkward - // file names. Collapse whitespace runs to a single underscore and - // drop any non-alphanumeric tail so the synthetic entry names stay - // POSIX-portable and round-trip through Extract → WriteFile. var sb = new StringBuilder(tag.Length); foreach (var c in tag) { if (char.IsLetterOrDigit(c)) @@ -263,8 +266,8 @@ private static string SanitiseSectionName(string tag) { else if (sb.Length > 0 && sb[^1] != '_') sb.Append('_'); } - var s = sb.ToString().TrimEnd('_'); - return s.Length == 0 ? "unnamed" : s; + var result = sb.ToString().TrimEnd('_'); + return result.Length == 0 ? "unnamed" : result; } private byte[] BuildMetadata() { @@ -275,37 +278,42 @@ private byte[] BuildMetadata() { bldr.Append("magic_tag=MFSM\n"); bldr.Append("magic_offset=0\n"); bldr.Append(CultureInfo.InvariantCulture, $"image_size={_data.Length}\n"); - if (this.FileIdCounter.HasValue) - bldr.Append(CultureInfo.InvariantCulture, $"file_id_counter={this.FileIdCounter.Value}\n"); + if (this.IsEmptyBootstrap) + bldr.Append("empty_bootstrap=true\n"); + if (this.FileFormatVersion is byte version) + bldr.Append(CultureInfo.InvariantCulture, + $"file_format_version={version >> 4}.{version & 0x0F}\n"); + if (this.MaxNodeId.HasValue) + bldr.Append(CultureInfo.InvariantCulture, $"max_node_id={this.MaxNodeId.Value}\n"); if (this.MetadataVersion.HasValue) bldr.Append(CultureInfo.InvariantCulture, $"metadata_version={this.MetadataVersion.Value}\n"); + if (this.NextSessionId.HasValue) + bldr.Append(CultureInfo.InvariantCulture, $"next_session_id={this.NextSessionId.Value}\n"); + if (this.MetaId.HasValue) + bldr.Append(CultureInfo.InvariantCulture, $"meta_id={this.MetaId.Value}\n"); bldr.Append(CultureInfo.InvariantCulture, $"section_count={_sections.Count}\n"); - for (var i = 0; i < _sections.Count; i++) { - var s = _sections[i]; + for (var i = 0; i < _sections.Count; ++i) { + var section = _sections[i]; bldr.Append(CultureInfo.InvariantCulture, - $"section[{i}]={s.Tag} offset={s.Offset} length={s.Length}\n"); + $"section[{i}]={section.Tag} offset={section.Offset} length={section.Length}\n"); } - bldr.Append("note=Partial R/O — outer envelope (signature + section index) "); - bldr.Append("only. NODE/EDGE/CHNK body decoding is version-specific and "); - bldr.Append("requires golden samples to validate honestly. File content "); - bldr.Append("lives on chunk servers and is not reachable from this image.\n"); + bldr.Append("note=Partial metadata-image view only. NODE/EDGE/CHNK body decoding is version-specific; "); + bldr.Append("file content lives on chunk servers and is not reachable from metadata.mfs alone.\n"); return Encoding.UTF8.GetBytes(bldr.ToString()); } - /// Returns the bytes that back the given entry (in-memory). + /// Returns the bytes backing a surfaced synthetic entry. public byte[] Extract(MooseFsEntry entry) { ArgumentNullException.ThrowIfNull(entry); return entry.Data; } - /// - /// Releases resources held by this instance. - /// + /// Releases resources held by this instance. public void Dispose() { } - /// One walked section from the master metadata stream. - /// The 8-byte ASCII tag (e.g. "NODE 1.0"), trimmed. - /// Byte offset of the section payload (after the 16-byte tag+length). - /// Length of the section payload in bytes. + /// One walked section from a 1.6+ master metadata stream. + /// Eight-byte section tag, for example NODE 1.0. + /// Byte offset of the section payload. + /// Payload length in bytes. public readonly record struct SectionEntry(string Tag, long Offset, long Length); } diff --git a/FileSystems/FileSystem.Nwfs386/FileSystem.Nwfs386.csproj b/FileSystems/FileSystem.Nwfs386/FileSystem.Nwfs386.csproj index 5f028fe08..333d0d1ef 100644 --- a/FileSystems/FileSystem.Nwfs386/FileSystem.Nwfs386.csproj +++ b/FileSystems/FileSystem.Nwfs386/FileSystem.Nwfs386.csproj @@ -6,6 +6,7 @@ + diff --git a/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs b/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs index b24d9862d..f3dc64cb2 100644 --- a/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs +++ b/FileSystems/FileSystem.Nwfs386/Nwfs386FormatDescriptor.cs @@ -1,142 +1,603 @@ #pragma warning disable CS1591 -using System.Globalization; -using System.Text; +using System.Buffers.Binary; using Compression.Registry; +using FileSystem.Nwfs; using static Compression.Registry.FormatHelpers; namespace FileSystem.Nwfs386; /// -/// Read-only descriptor for Novell NetWare 386 (NWFS386) raw partition dumps, -/// detected via the "NetW" ASCII prefix at offset 0. DOS partition type -/// 0x65. The on-disk format is FAT-like but proprietary; no parser is -/// attempted — the image is surfaced as a single opaque entry with -/// metadata.ini noting the partition-type hint. -/// -/// References: -/// -/// https://www.win.tue.nl/~aeb/partitions/partition_types-1.html — partition-type catalogue (0x65 = Novell NetWare) -/// https://en.wikipedia.org/wiki/NetWare_File_System — Wikipedia article -/// Novell NetWare 386 internal documentation — the on-disk format was never published -/// +/// Compatibility descriptor for the Novell NetWare 386 Traditional File System. +/// The actual reader/writer lives in FileSystem.Nwfs; this descriptor keeps +/// the historical Nwfs386 id/extensions while sharing that implementation. /// /// -/// Distinct from FileSystem.Nwfs, which detects via the -/// "HOTFIX00" magic at byte offset 0x4000 (sector-32-aligned NetWare -/// HOTFIX header). NWFS386 here covers raw NWFS partition dumps that -/// expose the "NetW" four-byte tag at offset 0 instead of the HOTFIX area. +/// +/// Fresh images and rewrites intentionally target the conservative profile the +/// shared writer can prove: one Traditional NetWare volume, one segment, DOS +/// namespace only, without compression, suballocation, migration, auditing or +/// special directory records. Unsupported legacy profiles remain readable where +/// the shared reader understands them, but mutation fails before the source is +/// touched. +/// +/// +/// Novell documents MBR partition type 0x65 as a Traditional NetWare +/// partition. The detailed on-disk structures are reconstructed from public +/// reverse-engineering/reference implementations; no third-party implementation +/// code is copied here. +/// /// -public sealed class Nwfs386FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { - /// - /// Gets the id. - /// +public sealed class Nwfs386FormatDescriptor : + IFormatDescriptor, + IArchiveFormatOperations, + IArchiveInMemoryExtract, + IArchiveCreatable, + IArchiveModifiable, + IArchiveDefragmentable, + IArchiveShrinkable, + IFormatOptionsSchema, + ILayoutOptimizable { + + private const long MaxManagedImageBytes = 512L * 1024 * 1024; + private const int SectorSize = 512; + private const int DirectoryEntryBytes = 128; + private const int FatEntryBytes = 8; + private const int VolumeAreaBytes = 4 * 16384; + private const uint NoBlock = 0xFFFFFFFF; + private const uint DirIdAvailable = 0xFFFFFFFF; + private const uint DirIdGrantOrDeleted = 0xFFFFFFFE; + private const uint DirIdVolumeInfo = 0xFFFFFFFD; + private const uint HighestSpecialDirectoryId = 0xFFFFFF00; + private const uint AttributeDirectory = 0x10; + private const uint AttributeArchive = 0x20; + + private static readonly int[] SupportedBlockSizes = + [1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144]; + public string Id => "Nwfs386"; - /// - /// Gets the display name. - /// - public string DisplayName => "NWFS386 (Novell NetWare 386 raw)"; - /// - /// Gets the category. - /// + public string DisplayName => "NWFS386 (Novell Traditional NetWare filesystem)"; public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest; - /// - /// Gets the default extension. - /// + FormatCapabilities.CanList | + FormatCapabilities.CanExtract | + FormatCapabilities.CanCreate | + FormatCapabilities.CanModify | + FormatCapabilities.CanTest | + FormatCapabilities.SupportsMultipleEntries | + FormatCapabilities.SupportsDirectories; + public string DefaultExtension => ".nwfs386"; - /// - /// Gets the extensions. - /// public IReadOnlyList Extensions => [".nwfs386", ".nw386"]; - /// - /// Gets the compound extensions. - /// public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// - public IReadOnlyList MagicSignatures => [ - // "NetW" ASCII at offset 0 — proprietary marker present in some NWFS386 - // raw partition dumps. Confidence 0.60 — short ASCII string at fixed - // offset, not from a published vendor spec. - new([0x4E, 0x65, 0x74, 0x57], Offset: 0, Confidence: 0.60), - ]; - /// - /// Gets the methods. - /// + + // The real HOTFIX00 signatures are already owned by FileSystem.Nwfs. + // Registering them twice would make magic routing depend on registry order. + // Nwfs386 is therefore an extension-routed compatibility id, exactly like + // other aliases in the repository that share one physical format. + public IReadOnlyList MagicSignatures => []; + public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// - public string Description => "Novell NetWare 386 raw partition — opaque single-entry surface."; - - /// - /// Lists the entries in the supplied container. - /// + public string Description => + "Novell NetWare 386 Traditional filesystem. R/W rebuilds are limited to the " + + "single-segment DOS-namespace profile without compression/suballocation; " + + "advanced legacy profiles are rejected before mutation."; + + public IReadOnlyList OptionsSchema => + [ + new( + "BlockSize", + "Block size", + FormatOptionKind.Integer, + "4096", + SupportedBlockSizes.Select(static value => value.ToString(System.Globalization.CultureInfo.InvariantCulture)).ToArray(), + "Allocation block size in bytes. Changing it rebuilds the volume; existing data is preserved byte-for-byte."), + new( + "VolumeName", + "Volume name", + FormatOptionKind.String, + "SYS", + Description: "NetWare volume name (1-19 ASCII characters).") + ]; + public List List(Stream stream, string? password) { - var entries = new List(); - byte[] image; - try { - image = ReadAll(stream); - } catch { - entries.Add(new ArchiveEntryInfo(0, "FULL.nwfs386", 0, 0, "stored", false, false, null)); - entries.Add(new ArchiveEntryInfo(1, "metadata.ini", 0, 0, "stored", false, false, null)); - return entries; + var opened = Open(stream); + var result = new List(); + var index = 0; + foreach (var item in opened.Volume.List()) + result.Add(new ArchiveEntryInfo( + index++, + item.Path, + item.Length, + item.Length, + "stored", + item.IsDirectory, + false, + null)); + return result; + } + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + var opened = Open(stream); + Directory.CreateDirectory(outputDir); + + foreach (var item in opened.Volume.List()) { + if (item.IsDirectory) continue; + if (files is { Length: > 0 } && !MatchesFilter(item.Path, files)) continue; + WriteFile(outputDir, item.Path, opened.Volume.Read(item)); } + } + + public void ExtractEntry(Stream input, string entryName, Stream output, string? password) { + ArgumentNullException.ThrowIfNull(output); + var opened = Open(input); + var item = opened.Volume.List().FirstOrDefault(candidate => + !candidate.IsDirectory && + candidate.Path.Equals(entryName, StringComparison.OrdinalIgnoreCase)); - entries.Add(new ArchiveEntryInfo(0, "FULL.nwfs386", image.LongLength, image.LongLength, "stored", false, false, null)); - entries.Add(new ArchiveEntryInfo(1, "metadata.ini", 0, 0, "stored", false, false, null)); - return entries; + if (item is null) + throw new FileNotFoundException($"NWFS386 entry '{entryName}' was not found.", entryName); + + output.Write(opened.Volume.Read(item)); } - /// - /// Decodes the supplied input. - /// - public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - byte[] image; - try { - image = ReadAll(stream); - } catch { - WriteFile(outputDir, "metadata.ini", Encoding.UTF8.GetBytes("parse_status=partial\n")); - return; + public Stream OpenEntry(Stream archive, string entryName, string? password) { + var opened = Open(archive); + var item = opened.Volume.List().FirstOrDefault(candidate => + !candidate.IsDirectory && + candidate.Path.Equals(entryName, StringComparison.OrdinalIgnoreCase)); + + if (item is null) + throw new FileNotFoundException($"NWFS386 entry '{entryName}' was not found.", entryName); + + return new MemoryStream(opened.Volume.Read(item), writable: false); + } + + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + ArgumentNullException.ThrowIfNull(options); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("NWFS386 creation requires a writable, seekable stream.", nameof(output)); + + var blockSize = options.GetOptionInt( + "BlockSize", + options.GetOptionInt("ClusterSize", 4096)); + var volumeName = options.GetOption("VolumeName", "SYS"); + + var files = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in inputs) { + if (input.IsDirectory) continue; + var name = NormalizePath(input.ArchiveName); + if (name.Length == 0) + throw new InvalidDataException("NWFS386 file entries need a non-empty path."); + files[name] = input.ReadContent(); } - var ok = image.Length >= 4 && image[0] == 'N' && image[1] == 'e' && image[2] == 't' && image[3] == 'W'; - WriteIfMatch(outputDir, "FULL.nwfs386", image, files); + var image = BuildImage(files, blockSize, volumeName); + output.Position = 0; + output.SetLength(0); + output.Write(image); + output.Position = 0; + } + + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(inputs); + var opened = OpenWritable(archive); + var files = SnapshotFiles(opened.Volume); + + foreach (var input in inputs) { + if (input.IsDirectory) continue; + var name = NormalizePath(input.ArchiveName); + if (name.Length == 0) + throw new InvalidDataException("NWFS386 file entries need a non-empty path."); + files[name] = input.ReadContent(); + } - var bldr = new StringBuilder(); - bldr.Append(CultureInfo.InvariantCulture, $"parse_status={(ok ? "ok" : "partial")}\n"); - bldr.Append("magic_text=NetW\n"); - bldr.Append("dos_partition_type=0x65\n"); - bldr.Append("note=NWFS386 raw partition. Layout proprietary; surfaced as opaque blob.\n"); - WriteIfMatch(outputDir, "metadata.ini", Encoding.UTF8.GetBytes(bldr.ToString()), files); + CommitRebuild(archive, files, opened.Volume.BlockSize, opened.Volume.VolumeName); } - private static void WriteIfMatch(string outputDir, string name, byte[] data, string[]? filter) { - if (filter != null && filter.Length > 0 && !MatchesFilter(name, filter)) return; - WriteFile(outputDir, name, data); + public void Remove(Stream archive, string[] entryNames) { + var opened = OpenWritable(archive); + var files = SnapshotFiles(opened.Volume); + var removals = (entryNames ?? []) + .Select(NormalizePath) + .Where(static name => name.Length > 0) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + foreach (var name in files.Keys.ToArray()) + if (removals.Any(removal => + name.Equals(removal, StringComparison.OrdinalIgnoreCase) || + name.StartsWith(removal + "/", StringComparison.OrdinalIgnoreCase))) + files.Remove(name); + + CommitRebuild(archive, files, opened.Volume.BlockSize, opened.Volume.VolumeName); + } + + public void Defragment(Stream archive) { + var opened = OpenWritable(archive); + CommitRebuild( + archive, + SnapshotFiles(opened.Volume), + opened.Volume.BlockSize, + opened.Volume.VolumeName); + } + + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(output); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("NWFS386 shrink requires a writable, seekable target stream.", nameof(output)); + + var opened = OpenWritable(input); + var files = SnapshotFiles(opened.Volume); + var rebuilt = BuildImage(files, opened.Volume.BlockSize, opened.Volume.VolumeName); + VerifyImage(rebuilt, files, opened.Volume.BlockSize, opened.Volume.VolumeName); + + output.Position = 0; + output.SetLength(0); + if (rebuilt.LongLength >= opened.Image.LongLength) + output.Write(opened.Image); + else + output.Write(rebuilt); + output.Position = 0; } - private const int HeaderReadCap = 64 * 1024; + public LayoutAnalysis AnalyzeLayout(Stream image) { + var opened = OpenWritable(image); + var files = opened.Volume.List().Where(static item => !item.IsDirectory).ToArray(); + + static long Slack(IEnumerable items, int blockSize) + => items.Sum(item => item.Length == 0 + ? 0 + : ((item.Length + blockSize - 1) / blockSize) * blockSize - item.Length); + + var currentSlack = Slack(files, opened.Volume.BlockSize); + var optimal = SupportedBlockSizes + .Select(blockSize => (BlockSize: blockSize, Slack: Slack(files, blockSize))) + .MinBy(static candidate => (candidate.Slack, candidate.BlockSize)); + + return new LayoutAnalysis { + ImageSize = opened.Image.LongLength, + CurrentUnitSize = opened.Volume.BlockSize, + CurrentSlackBytes = currentSlack, + OptimalUnitSize = optimal.BlockSize, + OptimalSlackBytes = optimal.Slack, + RequiresRebuild = optimal.BlockSize == opened.Volume.BlockSize + ? [] + : ["BlockSize"], + Notes = [ + "Optimization compares allocation slack for the plain NWFS386 profile; changing block size is a verified rebuild.", + "Compression, suballocation, alternate namespaces and multi-segment volumes are deliberately outside the writable profile." + ], + }; + } + + public void RebuildStreaming(Stream source, Stream target, LayoutRebuildOptions options) { + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(options); + if (!target.CanWrite || !target.CanSeek) + throw new ArgumentException("NWFS386 layout rebuild requires a writable, seekable target stream.", nameof(target)); + if (options.MakeSparse || options.DeduplicateWithLinks) + throw new NotSupportedException("NWFS386 layout rebuild does not support sparse-file or hard-link transforms."); + + var opened = OpenWritable(source); + var blockSize = options.UnitSize > 0 ? options.UnitSize : opened.Volume.BlockSize; + var volumeName = opened.Volume.VolumeName; - private static byte[] ReadAll(Stream stream) { - using var ms = new MemoryStream(); - var buf = new byte[8192]; - int read; - while (ms.Length < HeaderReadCap && (read = stream.Read(buf, 0, buf.Length)) > 0) - ms.Write(buf, 0, read); - return ms.ToArray(); + if (options.Parameters is not null) { + if (options.Parameters.TryGetValue("ClusterSize", out var clusterText) && + int.TryParse(clusterText, System.Globalization.CultureInfo.InvariantCulture, out var clusterSize)) + blockSize = clusterSize; + if (options.Parameters.TryGetValue("BlockSize", out var blockText) && + int.TryParse(blockText, System.Globalization.CultureInfo.InvariantCulture, out var explicitBlockSize)) + blockSize = explicitBlockSize; + if (options.Parameters.TryGetValue("VolumeName", out var explicitVolumeName)) + volumeName = explicitVolumeName; + } + + var files = SnapshotFiles(opened.Volume); + var rebuilt = BuildImage(files, blockSize, volumeName); + VerifyImage(rebuilt, files, blockSize, volumeName); + + target.Position = 0; + target.SetLength(0); + target.Write(rebuilt); + target.Position = 0; + options.OnProgress?.Invoke(opened.Image.LongLength, opened.Image.LongLength); + } + + private static OpenedVolume Open(Stream stream) { + var image = ReadImage(stream); + var volume = NwfsReader.TryOpen(image) + ?? throw new InvalidDataException("The stream is not a readable NWFS386 Traditional NetWare volume."); + return new OpenedVolume(image, volume); + } + + private static OpenedVolume OpenWritable(Stream stream) { + var opened = Open(stream); + ValidatePlainWritableProfile(opened.Image, opened.Volume); + return opened; + } + + private static byte[] ReadImage(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead || !stream.CanSeek) + throw new ArgumentException("NWFS386 requires a readable, seekable stream.", nameof(stream)); + if (stream.Length > MaxManagedImageBytes) + throw new NotSupportedException( + $"NWFS386's shared managed reader currently caps images at {MaxManagedImageBytes:N0} bytes."); + + stream.Position = 0; + var image = new byte[checked((int)stream.Length)]; + stream.ReadExactly(image); + return image; + } + + private static Dictionary SnapshotFiles(NwfsReader volume) { + var files = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var item in volume.List().Where(static item => !item.IsDirectory)) { + if (!files.TryAdd(item.Path, volume.Read(item))) + throw new NotSupportedException($"NWFS386 writable profile rejects duplicate path '{item.Path}'."); + } + return files; } + + private static byte[] BuildImage( + IReadOnlyDictionary files, + int blockSize, + string volumeName) { + if (string.IsNullOrWhiteSpace(volumeName) || volumeName.Length > 19 || + volumeName.Any(static ch => ch is < ' ' or > '~')) + throw new InvalidDataException("NWFS386 volume name must be 1-19 printable ASCII characters."); + + var writer = new NwfsWriter { + BlockSize = blockSize, + VolumeName = volumeName, + }; + + foreach (var (path, data) in files.OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase)) + writer.AddFile(path, data); + + return writer.Build(); + } + + private static void CommitRebuild( + Stream archive, + IReadOnlyDictionary files, + int blockSize, + string volumeName) { + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("NWFS386 mutation requires a readable, writable, seekable stream.", nameof(archive)); + + var rebuilt = BuildImage(files, blockSize, volumeName); + VerifyImage(rebuilt, files, blockSize, volumeName); + + archive.Position = 0; + archive.SetLength(0); + archive.Write(rebuilt); + archive.Flush(); + archive.Position = 0; + } + + private static void VerifyImage( + byte[] image, + IReadOnlyDictionary expectedFiles, + int blockSize, + string volumeName) { + var verify = NwfsReader.TryOpen(image) + ?? throw new InvalidOperationException("NWFS386 writer produced an image its own reader rejects."); + if (verify.BlockSize != blockSize) + throw new InvalidOperationException( + $"NWFS386 rebuild changed block size ({blockSize} -> {verify.BlockSize})."); + if (!verify.VolumeName.Equals(volumeName, StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException( + $"NWFS386 rebuild changed volume name ('{volumeName}' -> '{verify.VolumeName}')."); + + var actual = SnapshotFiles(verify); + if (!actual.Keys.Order(StringComparer.OrdinalIgnoreCase) + .SequenceEqual(expectedFiles.Keys.Order(StringComparer.OrdinalIgnoreCase), StringComparer.OrdinalIgnoreCase)) + throw new InvalidOperationException("NWFS386 rebuild changed the live file set."); + + foreach (var (name, expected) in expectedFiles) + if (!actual.TryGetValue(name, out var bytes) || !bytes.AsSpan().SequenceEqual(expected)) + throw new InvalidOperationException($"NWFS386 rebuild changed the bytes of '{name}'."); + } + + private static void ValidatePlainWritableProfile(byte[] image, NwfsReader volume) { + if (!TryLocateVolume(image, out var layout)) + throw new NotSupportedException("NWFS386 mutation requires a single Traditional NetWare volume."); + + var root = layout.RootDirectoryBlock; + var guard = 0; + var sawVolumeInfo = false; + while (root != NoBlock && guard++ < 1 << 20) { + var blockOffset = layout.DataAreaOffset + (long)(root - layout.FirstSegmentBlock) * layout.BlockSize; + if (blockOffset < 0 || blockOffset + layout.BlockSize > image.LongLength) + throw new NotSupportedException("NWFS386 mutation refused a directory chain outside the supported volume bounds."); + + var entriesPerBlock = layout.BlockSize / DirectoryEntryBytes; + for (var i = 0; i < entriesPerBlock; ++i) { + var offset = checked((int)(blockOffset + (long)i * DirectoryEntryBytes)); + var entry = image.AsSpan(offset, DirectoryEntryBytes); + var parent = BinaryPrimitives.ReadUInt32LittleEndian(entry); + + if (parent == DirIdAvailable) + continue; + + if (parent == DirIdVolumeInfo) { + sawVolumeInfo = true; + // ROOT/ROOT3X: byte 23 is VolumeFlags. Writer-created plain volumes keep + // auditing, suballocation, compression, migration, NDS/trustee flags off. + if (entry[23] != 0) + throw new NotSupportedException( + "NWFS386 mutation is limited to volumes without compression, suballocation, migration, auditing or NDS flags."); + continue; + } + + // -2 is used by historical implementations for grant/deleted records; + // other high unsigned ids are special directory records. The shared + // writer does not reproduce those structures, so refuse rather than drop them. + if (parent == DirIdGrantOrDeleted || parent >= HighestSpecialDirectoryId) + throw new NotSupportedException( + "NWFS386 mutation refused special/deleted/trustee directory records that the plain writer cannot preserve."); + + // DOS namespace is zero; any other namespace may carry a parallel name + // or metadata record the plain writer cannot round-trip. + if (entry[10] != 0 || entry[9] != 0) + throw new NotSupportedException( + "NWFS386 mutation is limited to the primary DOS namespace without special directory-entry flags."); + + var attributes = BinaryPrimitives.ReadUInt32LittleEndian(entry[4..]); + if ((attributes & ~(AttributeDirectory | AttributeArchive)) != 0) + throw new NotSupportedException( + "NWFS386 mutation refused file attributes outside the plain writer's preserved subset."); + + if ((attributes & AttributeDirectory) == 0 && + BinaryPrimitives.ReadUInt32LittleEndian(entry[104..]) != 0) + throw new NotSupportedException( + "NWFS386 mutation refused a volume containing salvage/deleted-file records."); + } + + var fatOffset = layout.DataAreaOffset + + (long)(root - layout.FirstSegmentBlock) * FatEntryBytes + + 4; + if (fatOffset < 0 || fatOffset + 4 > image.LongLength) + throw new NotSupportedException("NWFS386 mutation refused a truncated directory FAT chain."); + root = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan((int)fatOffset, 4)); + } + + if (!sawVolumeInfo) + throw new NotSupportedException("NWFS386 mutation requires the root volume-information record."); + + var items = volume.List(); + ValidateFileChains(image, layout, items); + + // Empty directories cannot be represented by the shared writer yet. Refuse + // such a source for rewrite operations so an edit never makes one disappear. + var files = items.Where(static item => !item.IsDirectory).Select(static item => item.Path).ToArray(); + foreach (var directory in items.Where(static item => item.IsDirectory)) + if (!files.Any(file => file.StartsWith(directory.Path + "/", StringComparison.OrdinalIgnoreCase))) + throw new NotSupportedException( + $"NWFS386 mutation refused empty directory '{directory.Path}' because the current writer cannot preserve it."); + } + + private static void ValidateFileChains( + byte[] image, + PlainLayout layout, + IReadOnlyList items) { + foreach (var item in items.Where(static item => !item.IsDirectory)) { + var expectedBlocks = item.Length == 0 + ? 0L + : (item.Length + layout.BlockSize - 1) / layout.BlockSize; + + if (expectedBlocks == 0) { + if (item.FirstBlock != NoBlock) + throw new NotSupportedException( + $"NWFS386 mutation refused empty file '{item.Path}' with an allocated data chain."); + continue; + } + + var block = item.FirstBlock; + for (long expectedIndex = 0; expectedIndex < expectedBlocks; ++expectedIndex) { + if (block == NoBlock) + throw new NotSupportedException( + $"NWFS386 mutation refused truncated/sparse FAT chain for '{item.Path}'."); + + var relativeBlock = (long)block - layout.FirstSegmentBlock; + var fatOffset = layout.DataAreaOffset + relativeBlock * FatEntryBytes; + if (relativeBlock < 0 || fatOffset < 0 || fatOffset + FatEntryBytes > image.LongLength) + throw new NotSupportedException( + $"NWFS386 mutation refused out-of-range FAT chain for '{item.Path}'."); + + var fatIndex = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan((int)fatOffset, 4)); + if (fatIndex != expectedIndex) + throw new NotSupportedException( + $"NWFS386 mutation refused sparse/non-linear FAT indexing for '{item.Path}'."); + + block = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan((int)fatOffset + 4, 4)); + } + + if (block != NoBlock) + throw new NotSupportedException( + $"NWFS386 mutation refused an overlong FAT chain for '{item.Path}'."); + } + } + + private static bool TryLocateVolume(ReadOnlySpan image, out PlainLayout layout) { + layout = default; + var hotfix = FindHotfix(image); + if (hotfix < 0 || hotfix + 28 > image.Length) + return false; + + var redirectionSectors = + BinaryPrimitives.ReadUInt32LittleEndian(image.Slice(checked((int)hotfix + 24), 4)); + var volumeArea = hotfix + (long)redirectionSectors * SectorSize; + if (volumeArea < 0 || volumeArea + 32 + 60 > image.Length) + return false; + if (!image.Slice((int)volumeArea, 16).SequenceEqual("NetWare Volumes\0"u8)) + return false; + + var count = BinaryPrimitives.ReadUInt32LittleEndian(image.Slice((int)volumeArea + 16, 4)); + if (count != 1) + return false; + + var entry = image.Slice((int)volumeArea + 32, 60); + + // VOLUME_TABLE_ENTRY.LastVolumeSegment == 0 is the single-segment profile. + if (BinaryPrimitives.ReadUInt32LittleEndian(entry[16..]) != 0) + return false; + + var firstSegmentBlock = BinaryPrimitives.ReadUInt32LittleEndian(entry[36..]); + var blockValue = BinaryPrimitives.ReadUInt32LittleEndian(entry[44..]); + if (blockValue == 0 || (256u * 1024u) % blockValue != 0) + return false; + var blockSize = checked((int)(256u * 1024u / blockValue)); + if (!SupportedBlockSizes.Contains(blockSize)) + return false; + + var rootDirectoryBlock = BinaryPrimitives.ReadUInt32LittleEndian(entry[48..]); + var dataAreaOffset = volumeArea + VolumeAreaBytes; + if (dataAreaOffset < 0 || dataAreaOffset >= image.Length) + return false; + + layout = new PlainLayout(dataAreaOffset, blockSize, firstSegmentBlock, rootDirectoryBlock); + return true; + } + + private static long FindHotfix(ReadOnlySpan image) { + const int partitionTableOffset = 446; + const byte netWare386PartitionType = 0x65; + const long hotfixOffsetInPartition = 0x4000; + + if (image.Length >= 512 && image[510] == 0x55 && image[511] == 0xAA) { + for (var i = 0; i < 4; ++i) { + var entry = image.Slice(partitionTableOffset + i * 16, 16); + if (entry[4] != netWare386PartitionType) continue; + + var partitionStart = + (long)BinaryPrimitives.ReadUInt32LittleEndian(entry[8..]) * SectorSize; + var at = partitionStart + hotfixOffsetInPartition; + if (at >= 0 && at + 8 <= image.Length && + image.Slice((int)at, 8).SequenceEqual("HOTFIX00"u8)) + return at; + } + } + + return hotfixOffsetInPartition + 8 <= image.Length && + image.Slice((int)hotfixOffsetInPartition, 8).SequenceEqual("HOTFIX00"u8) + ? hotfixOffsetInPartition + : -1; + } + + private static string NormalizePath(string path) + => (path ?? string.Empty).Replace('\\', '/').Trim('/'); + + private readonly record struct OpenedVolume(byte[] Image, NwfsReader Volume); + private readonly record struct PlainLayout( + long DataAreaOffset, + int BlockSize, + uint FirstSegmentBlock, + uint RootDirectoryBlock); } diff --git a/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md b/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md new file mode 100644 index 000000000..35beb1904 --- /dev/null +++ b/FileSystems/FileSystem.OneFs/FORMAT-NOTES.md @@ -0,0 +1,232 @@ +# Dell PowerScale / Isilon OneFS raw-media notes + +## Scope + +`FileSystem.OneFs` is an **offline raw-image** handler. It is not a client for a +running PowerScale cluster and must not silently substitute OneFS REST, SMB, NFS +or SSH operations for the repository's `Stream`-based image contracts. + +The September 2026 review found substantially more public structure than the +original Stage-0 stub assumed, but still not enough byte-level serialization to +reconstruct or mutate a standalone OneFS data drive safely. + +## What is publicly established + +Dell's own architecture material establishes these invariants: + +- OneFS presents a **single global namespace across the cluster**. +- Files and directories are identified by 64-bit LINs (logical inode numbers). +- The **superblock exists at multiple fixed block addresses on every drive**. + It contains addresses for the LIN master, which leads to the root of the LIN + B+ tree. Dell publishes this traversal concept but not the fixed block numbers + or byte layout of the superblock. +- The LIN B+ tree maps LINs to **mirrored inode disk addresses**. `isi get` + examples render an inode address as a node, logical drive, byte/block address + and inode size, for example `1,9,1054720:512`. +- File IFM/metatrees map logical file blocks into **protection groups**; directory + DFM trees hold names and namespace relationships. +- Data addresses shown by `isi get -DD` use the same distributed shape and can + identify sparse regions as `0,0,0:8192`. +- Each data disk is assigned a GUID and logical drive number and is divided into + **32 MiB cylinder groups** consisting of **8 KiB filesystem blocks**. That is + 4096 filesystem blocks per cylinder group. +- Every cylinder group tracks, with a bitmap, whether blocks are used for file + data, inodes, or other metadata. Public material does not define the bitmap's + raw offset or serialization. +- Layout is managed by BAM (Block Allocation Manager); local disk allocation is + delegated to the LBM (Local Block Manager), while remote block operations go + through the RBM. +- Safe writes are distributed transactions. BAM Safe Write builds a write plan, + then OneFS uses two-phase commit across participating nodes and their journals. + A correct offline writer therefore cannot update a single drive in isolation + and pretend cluster consistency was preserved. +- OneFS is based on UFS / FreeBSD heritage, but its `/ifs` storage layer replaces + or extends the ordinary standalone-UFS allocation model with distributed + BAM/LBM/RBM addressing and protection. UFS ancestry is useful historical + context, not a license to route an isolated OneFS data disk through `UfsReader`. + +## `newfs_efs` and the UFS-looking format logs + +A deeper search found old reimage logs where `newfs_efs` prints classic BSD-newfs +style geometry such as an 8 KiB block size, 1 KiB fragments, cylinder groups and +backup-superblock locations. That specific log formats `/dev/imdd5a`, however: +IMDD is OneFS's **Mirrored Device Driver for system volumes** such as root/`/var`, +not proof that an `/ifs` data disk is a normal UFS volume. + +Separately, Dell's reimage documentation says `isi_initial_newfs` then formats the +`/ifs` partition on all drives and announces `Proceeding with v16 newfs...` while +creating a linked journal. Combined with Dell's documented 32 MiB/8 KiB data-disk +geometry, this confirms substantial BSD/UFS lineage in the formatter and physical +organization, but it still does not publish the proprietary `/ifs` superblock, +allocation bitmap, LIN tree or transaction record layouts. + +The practical conclusion is narrower than the old stub's wording: **OneFS is +UFS-derived, but an isolated OneFS data drive is not established as a complete, +generic UFS filesystem image.** + +## Patent-level structural evidence + +Several Isilon patents predate Dell's acquisition and describe implementation +behavior in enough detail to sharpen the model without supplying a usable raw +serialization: + +- US8214400B2 / US7797283 describes a distributed mirrored index tree. A + superblock contains a header plus block-address (`baddr`) pointers to copies of + the tree root. The header includes version/tree-height information; unused + pointers may be zero. Inner nodes keep sorted keys/offsets at one end and + mirrored child pointers at the other, leaving free space between them. +- US7937421B2 / application 20040153479 describes BAM layout selection and names + `ifs_lbn_t`, `ifs_devid_t` and `ifs_baddr_t`; disk/block allocation itself is + delegated to LBM. +- US7743033 describes BAM as working with and/or instead of BSD UFS, which is + consistent with Dell's later description of OneFS as UFS-based while using a + distributed allocation layer. + +These patents are used only as behavioral/specification evidence. Their +pseudocode and expressive implementation are **not copied or translated**. + +## Other behavioral oracles found + +Public Dell command output gives useful consistency checks for a future reader: + +- `isi get -D` exposes inode version, directory version, inode revision, mirror + count, parent LIN/hash, physical-block counts and rendered IFS inode addresses. +- `isi get -DD` exposes protection groups and distributed physical data addresses. +- Dell documents OneFS NFS file-handle chunks as little-endian; a file handle can + carry the 64-bit LIN. This is evidence about an exported protocol encoding, not + enough to assume every raw-disk integer uses the same encoding. +- OneFS release/recovery logs contain `Bad Magic in superblock`, proving a + superblock magic check exists in at least relevant OneFS metadata/journal paths. + The expected value and authoritative raw-drive offset remain unverified. +- Commercial recovery reports describe scanning **all drive images** for OneFS + inode and directory-entry copies, then combining those results cluster-wide. + That independently reinforces the multi-drive reconstruction requirement. + +A public GitHub repository named `jji0717/cp-migrate` contains source that refers +to proprietary OneFS headers such as `ifs/ifs_types.h`, `ifs/ifm/ifm_dinode.h` +and BAM APIs. The repository declares **no license** and does not contain the +missing proprietary header definitions. It is therefore treated only as a source +of behavioral/symbol-name clues; no code, layout or expressive structure from it +is reused here. + +## Genuine-media oracle path + +Dell publicly distributes a OneFS Simulator specifically for testing. Dell's own +community linked the 9.5.0.0 simulator ZIP, and the simulator installation guide +states that the ZIP contains an OVA. Independent inspection of that OVA reports a +small sparse appliance with 22 VMDKs: one 16 GiB system disk, one 512 MiB device, +and twenty 5 GiB SCSI data disks. Those data VMDKs are the best available oracle +for the next raw-format step because they are genuine Dell-authored OneFS media, +not synthetic bytes invented by this repository. + +The intended clean-room oracle procedure is: + +1. obtain the simulator from Dell's own download channel; +2. extract the OVA/VMDKs without redistributing them; +3. identify which 5 GiB VMDKs are initialized `/ifs` data drives; +4. reconstruct each sparse VMDK as logical raw sectors using this repository's + existing VMDK reader; +5. compare the same 8 KiB block positions across many drives and fresh simulator + clusters to find the repeated fixed-address superblock candidates; +6. change one controlled cluster property at a time, then diff those candidates; +7. validate any inferred magic/version/root-address fields against live `isi get`, + cluster upgrade behavior and more than one simulator version before adding a + detector or parser. + +This run could verify the public simulator packaging and disk inventory, but the +execution environment could not resolve `dl.dell.com`, so no simulator bytes were +used and no claim about the missing raw offsets or magic is made here. + +## What remains unknown at byte level + +No authoritative publication or independently licensed implementation found in +the review defines enough raw serialization for safe traversal or mutation. In +particular, the review still does **not** establish: + +- a fixed offset-zero `"OneFS"` or `"ONEF"` drive signature; +- the actual superblock magic, exact fixed superblock block addresses, checksum, + header size, field offsets or version compatibility rules; +- the LIN-master record serialization; +- inode and B+/B* tree record encodings across supported OneFS generations; +- the cylinder-group allocation bitmap location, width, bit meanings or update + rules; +- protection-group/parity record serialization needed to reconstruct or relocate + file content; +- journal and two-phase-commit record formats needed to make crash-consistent + writes; +- an offline single-drive checker that can act as an independent write oracle. + +The previous descriptor's `"OneFS"` / `"ONEF"` offset-zero magic therefore still +has no acceptable reference and remains removed. + +## Capability decision + +| Capability | State | Reason | +| --- | --- | --- | +| list / extract | limited | synthetic metadata + byte-exact raw-image stream only | +| geometry analysis | **supported** | Dell fixes blocks at 8 KiB and groups at 32 MiB; no payload parsing required | +| create | blocked | no public raw-media writer specification/oracle | +| modify / R/W | blocked | namespace, allocation, protection and journal state are cluster-wide | +| purge | blocked | cannot remove live namespace entries and atomically update distributed metadata | +| wipe | blocked | bitmap location/encoding is unknown, so no raw block can be proven free safely | +| defrag | blocked | relocation requires IFM/protection-group/allocation updates and distributed commit | +| shrink | blocked | no proven tail ownership or supported smaller physical geometry | +| layout rebuild | blocked | fixed geometry is known; a valid alternate creator/rebuilder is not | +| compact | blocked | composite of defrag/layout/shrink, all rewriting steps remain blocked | + +`OneFsFormatDescriptor` therefore implements `ILayoutOptimizable` **only for +analysis**. `AnalyzeLayout` reports the documented 8 KiB unit and 32 MiB cylinder +group. It does not estimate unknown slack and does not override `RebuildStreaming`. +The support matrix consequently keeps Layout and Compact unadvertised. + +This is deliberately stricter than implementing no-op maintenance interfaces: +the support matrix treats an advertised rewriting interface as a working verb. +Returning success while changing nothing—or guessing that silent regions are +free—would be operationally dishonest and potentially destructive. + +## References + +Primary / official material: + +- Dell Technologies Info Hub, **OneFS Metadata**: + https://infohub.delltechnologies.com/en-us/p/onefs-metadata/ +- Dell Technologies Info Hub, **OneFS architectural overview** (32 MiB cylinder + groups, 8 KiB blocks, allocation bitmaps, BAM/LBM/RBM, safe writes): + https://infohub.delltechnologies.com/en-us/l/high-availability-and-data-protection-with-dell-powerscale-scale-out-nas/onefs-architectural-overview-1/1/ +- Dell Technologies Info Hub, **OneFS data inlining / performance** (fixed-address + superblocks and LIN-master traversal): + https://infohub.delltechnologies.com/en-us/l/dell-powerscale-onefs-data-reduction-and-storage-efficiency/performance-854/ +- Dell PowerScale OneFS Technical Overview, **File system structure**: + https://infohub.delltechnologies.com/en-nz/l/dell-powerscale-onefs-technical-overview/file-system-structure/1/ +- Dell PowerScale reimage procedure, `isi_initial_newfs` / `/ifs` formatting + behavior (Dell-hosted/support material; mirrored copies may require support + access depending on region/account). +- Dell KB, **How to obtain a LIN from an NFS file handle** (little-endian exported + LIN encoding): + https://www.dell.com/support/kbdoc/en-us/000019498/isilon-onefs-how-to-obtain-a-lin-from-an-nfs-file-handle +- Dell OneFS Simulator Installation Guide: + https://www.dell.com/support/manuals/en-us/isilon-onefs/ifs_pub_onefs_simulator_guide/installing-onefs-simulator +- Dell community, **Upgrading OneFS Simulator** (Dell-hosted 9.5 simulator link): + https://www.dell.com/community/en/conversations/isilon/upgrading-onefs-simulator/647f8bdcf4ccf8a8dec07295 + +Implementation-independent patent evidence: + +- US8214400B2, **Systems and methods for maintaining distributed data**: + https://patents.google.com/patent/US8214400B2/en +- US7937421B2, **Systems and methods for restriping files in a distributed file + system**: + https://patents.google.com/patent/US7937421B2/en +- US7743033, **Systems and methods for providing a distributed file system + utilizing metadata to track information about data stored throughout the + system**. + +## Licensing / implementation provenance + +Dell documentation and patents are used only to determine factual architecture, +public behavior, constants/geometry and acceptance boundaries. Patent pseudocode +is not copied or translated. The unlicensed `cp-migrate` repository is not a code +source. Dell simulator binaries are not committed or redistributed; when +available they are suitable only as a behavioral/raw-media oracle. No external +implementation code, comments, naming structure or control flow is reproduced. +The implementation remains original managed C# under this repository's license, +with no new dependency. diff --git a/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs b/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs index e9f8a976a..bd2fbf68f 100644 --- a/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs +++ b/FileSystems/FileSystem.OneFs/OneFsFormatDescriptor.cs @@ -1,160 +1,166 @@ #pragma warning disable CS1591 using Compression.Registry; -using Compression.Registry.Streaming; using static Compression.Registry.FormatHelpers; namespace FileSystem.OneFs; /// -/// Stage 0 detection-only descriptor for Dell EMC Isilon OneFS LIN-tree -/// root images. Surfaces only a synthetic metadata.ini and the raw -/// image bytes; no real file-walk is attempted. -/// +/// Conservative inspection descriptor for Dell PowerScale / Isilon OneFS media. +/// +/// /// -/// Why R/O promotion is impossible (per CONTRIBUTING.md promotion gates): +/// OneFS is a distributed filesystem. Dell documents a global namespace, a LIN +/// B+ tree whose values are mirrored inode addresses on node/drive/block tuples, +/// and IFM metatrees that map logical file blocks into protection groups spread +/// through the cluster. A single raw drive therefore does not carry enough state +/// to reconstruct arbitrary files or safely rewrite their allocation metadata. +/// +/// +/// The public architecture does expose stable physical geometry: data disks are +/// divided into 32 MiB cylinder groups containing 8 KiB filesystem blocks, with +/// an allocation bitmap per group. This descriptor reports that geometry through +/// but deliberately inherits the interface's +/// refusing rebuild default. Analysis is therefore available without advertising +/// the repository's Layout or Compact maintenance verbs. /// -/// -/// -/// -/// No single-image content surface. OneFS is a clustered scale-out NAS -/// — every file is split into "protection groups" striped across drives and -/// nodes with FEC (Forward Error Correction, N+M:B layout, e.g. N+2:1). A -/// single drive/node image carries only one stripe; the file data cannot be -/// reconstructed without the peer nodes. A read-only reader from one image -/// can never return correct file bytes. -/// -/// -/// -/// -/// LIN tree is cluster-wide. The Logical Inode Number tree (the OneFS -/// metadata index) lives across nodes, not in a single superblock. There is -/// no per-image inode-to-block mapping to walk. -/// -/// -/// -/// -/// Proprietary on-disk format, no public specification. Dell EMC has -/// never published the OneFS on-disk format. No open-source reverse-engineered -/// reader exists. Without a spec we cannot honour the CONTRIBUTING rule -/// "never advertise capabilities you cannot prove against a real spec". -/// -/// -/// -/// -/// FreeBSD/UFS ancestry does NOT give us a UFS reader fallback. OneFS -/// runs on a FreeBSD-derived kernel, but the filesystem layer is entirely -/// proprietary — it is NOT FFS/UFS at the on-disk level. UFS1 places its -/// superblock magic 0x00011954 at offset 8192; OneFS images have the -/// ASCII "OneFS" tag at offset 0 and no UFS superblock. Routing OneFS -/// images through UfsReader would fail the magic check and (if forced) -/// return arbitrary bytes — the textbook mutual-compensation trap. -/// -/// -/// -/// /// -/// Conclusion: Stage-0 detection only. Surface the magic, raw bytes, -/// and a metadata.ini documenting the limitation. R/O promotion is -/// blocked on (a) Dell EMC publishing the spec and (b) a multi-node ingest -/// path — neither is in reach. +/// No Dell publication found during the implementation review defines the +/// historical "OneFS" or "ONEF" literals as fixed offset-zero raw +/// media signatures. Advertising those bytes caused unrelated files beginning +/// with either string to be identified as OneFS. Detection is consequently by +/// explicit selection or the unambiguous .onefs extension only. /// -/// -/// References: +/// +/// R/W and the destructive maintenance verbs remain deliberately unavailable. +/// Safe OneFS writes use distributed two-phase commit and per-node journals; +/// modifying allocation or tree state on one isolated drive would bypass that +/// protocol. The reader instead exposes documented geometry and architecture plus +/// the original image as a byte-exact bounded stream. +/// +/// +/// Clean-room references: /// -/// Dell EMC "PowerScale OneFS Technical Overview" whitepaper — high-level architecture only; no on-disk spec is published -/// https://en.wikipedia.org/wiki/OneFS_distributed_file_system — Wikipedia article +/// https://infohub.delltechnologies.com/en-us/p/onefs-metadata/ +/// — LIN tree, inode mirrors, IFM/DFM B+ trees and protection metadata. +/// Dell, "High Availability and Data Protection with Dell +/// PowerScale Scale-Out NAS" — 32 MiB cylinder groups, 8 KiB blocks, +/// per-group allocation bitmaps and BAM/LBM architecture. +/// Dell Info Hub, OneFS data inlining / data reduction — +/// fixed-address superblocks and the superblock → LIN-master chain. +/// Isilon patents US8214400B2 and US7937421B2 — distributed +/// mirrored index-tree and BAM/LBM behavioural model. Patent material is used +/// only as an implementation-independent behavioural oracle. /// -/// -public sealed class OneFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations { +/// No external implementation code is copied or translated. +/// +/// +public sealed class OneFsFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, ILayoutOptimizable { - /// - /// Gets the id. - /// + /// Gets the registry id. public string Id => "OneFs"; - /// - /// Gets the display name. - /// + + /// Gets the display name. public string DisplayName => "Dell EMC Isilon OneFS"; - /// - /// Gets the category. - /// + + /// Gets the format category. public FormatCategory Category => FormatCategory.Archive; + /// - /// Gets the capabilities. + /// Gets the conservative inspection capabilities. Integrity testing is not + /// advertised: without a verified superblock/tree parser, successfully copying + /// an opaque image does not prove that the filesystem is structurally sound. /// public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest; - /// - /// Gets the default extension. - /// + FormatCapabilities.CanList | FormatCapabilities.CanExtract; + + /// Gets the conventional extension for explicitly supplied raw media. public string DefaultExtension => ".onefs"; - /// - /// Gets the extensions. - /// + + /// Gets the recognized extensions. public IReadOnlyList Extensions => [".onefs"]; - /// - /// Gets the compound extensions. - /// + + /// Gets compound extensions. public IReadOnlyList CompoundExtensions => []; + /// - /// Gets the magic signatures. - /// - public IReadOnlyList MagicSignatures => [ - // ASCII "OneFS" (5 bytes) at offset 0 — long tag form. - new("OneFS"u8.ToArray(), Offset: 0, Confidence: 0.90), - // ASCII "ONEF" (0x4F4E4546 BE) at offset 0 — short tag form. - new("ONEF"u8.ToArray(), Offset: 0, Confidence: 0.85), - ]; - /// - /// Gets the methods. + /// Gets fixed magic signatures. Dell does not publish an authoritative + /// offset-zero raw-media signature, so OneFS intentionally has none. /// + public IReadOnlyList MagicSignatures => []; + + /// Gets the pseudo-archive storage method. public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// + + /// Gets the tar compression format id. public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// + + /// Gets the algorithm family. public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// + + /// Gets a description of the intentionally limited inspection surface. public string Description => - "Dell EMC Isilon OneFS — Stage 0, detection-only — proprietary distributed/clustered FS, " + - "no single-image content surface (file data is FEC-striped across nodes). " + - "FreeBSD-derived kernel but filesystem layer is NOT UFS-compatible (no UFS1 superblock at 8192). " + - "No public on-disk spec; R/O promotion blocked. " + - "Magic 'OneFS' / 'ONEF' at offset 0 of LIN-tree root."; + "Dell PowerScale / Isilon OneFS — conservative single-image inspection with documented geometry analysis. " + + "OneFS data disks use 8 KiB blocks in 32 MiB cylinder groups, while the namespace and file metatrees resolve " + + "through mirrored node/drive/block addresses and distributed protection groups. Fixed-address superblocks point " + + "toward the LIN master, but Dell does not publish the byte-level superblock/tree/allocation serialization or an " + + "authoritative offset-zero media magic. The historic 'OneFS'/'ONEF' signature claim was therefore removed. " + + "Integrity testing, R/W, purge, wipe, defrag, shrink and layout rebuild remain blocked until the raw structures " + + "and allocation/protection/journal updates can be proven against genuine media and an independent checker."; - /// - /// Lists the entries in the supplied container. - /// + /// Lists the two conservative inspection entries without reading the image payload. public List List(Stream stream, string? password) { - var r = new OneFsReader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo( - i, e.Name, e.Size, e.Size, "Stored", e.IsDirectory, false, null)).ToList(); + using var reader = new OneFsReader(stream, leaveOpen: true); + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo( + index, entry.Name, entry.Size, entry.Size, "Stored", entry.IsDirectory, false, null)).ToList(); } - /// - /// Decodes the supplied input. - /// + /// Extracts selected inspection entries through bounded streams. public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new OneFsReader(stream); - foreach (var e in r.Entries) { - if (e.IsDirectory) continue; - if (files != null && !MatchesFilter(e.Name, files)) continue; - WriteFile(outputDir, e.Name, r.Extract(e)); + using var reader = new OneFsReader(stream, leaveOpen: true); + foreach (var entry in reader.Entries) { + if (entry.IsDirectory) + continue; + if (files is { Length: > 0 } && !MatchesFilter(entry.Name, files)) + continue; + + using var source = reader.OpenEntry(entry); + using var target = CreateEntryFile(outputDir, entry.Name); + source.CopyTo(target); } } Stream IArchiveFormatOperations.OpenEntry(Stream archive, string entryName, string? password) { ArgumentNullException.ThrowIfNull(archive); ArgumentNullException.ThrowIfNull(entryName); - var r = new OneFsReader(archive); - var entry = r.Entries.FirstOrDefault(e => e.Name == entryName) - ?? throw new FileNotFoundException($"OneFS entry not found: {entryName}"); - var data = r.Extract(entry); - return new BoundedEntryStream(new MemoryStream(data, writable: false), data.Length, leaveOpen: false); + + using var reader = new OneFsReader(archive, leaveOpen: true); + var entry = reader.Entries.FirstOrDefault(candidate => + string.Equals(candidate.Name, entryName, StringComparison.Ordinal)) + ?? throw new FileNotFoundException($"OneFS entry not found: {entryName}", entryName); + return reader.OpenEntry(entry); + } + + /// + /// Reports the fixed physical geometry documented by Dell without reading or + /// interpreting proprietary allocation metadata. + /// + public LayoutAnalysis AnalyzeLayout(Stream image) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanSeek) + throw new ArgumentException("OneFS layout analysis requires a seekable stream.", nameof(image)); + + return new LayoutAnalysis { + ImageSize = image.Length, + CurrentUnitSize = OneFsReader.PhysicalBlockSize, + CurrentSlackBytes = 0, + OptimalUnitSize = OneFsReader.PhysicalBlockSize, + OptimalSlackBytes = 0, + Notes = [ + $"Documented fixed filesystem block size: {OneFsReader.PhysicalBlockSize} bytes.", + $"Documented cylinder-group size: {OneFsReader.CylinderGroupSize} bytes ({OneFsReader.BlocksPerCylinderGroup} blocks).", + "Each cylinder group has an allocation bitmap, but its raw location/encoding is not public; slack/free-space savings are therefore unknown and intentionally reported as zero.", + "Analysis only: rebuilding or patching OneFS geometry is unsupported and the Layout/Compact maintenance verbs remain unavailable.", + ], + }; } } diff --git a/FileSystems/FileSystem.OneFs/OneFsReader.cs b/FileSystems/FileSystem.OneFs/OneFsReader.cs index d97cf5aad..1a8db2384 100644 --- a/FileSystems/FileSystem.OneFs/OneFsReader.cs +++ b/FileSystems/FileSystem.OneFs/OneFsReader.cs @@ -1,120 +1,200 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; using System.Globalization; using System.Text; +using Compression.Registry.Streaming; namespace FileSystem.OneFs; /// -/// Stage 0 detection-only reader for Dell EMC Isilon OneFS LIN-tree root -/// images. OneFS is a clustered scale-out NAS — its single-image surface -/// is the LIN-tree root block, whose first bytes are the ASCII tag -/// "OneFS" (5 bytes, 0x4F 0x6E 0x65 0x46 0x53) or the short -/// "ONEF" tag (0x4F 0x4E 0x45 0x46 = 0x4F4E4546 BE int) used in -/// some node-local boot images. -/// +/// Conservative single-image inspection surface for Dell PowerScale / Isilon +/// OneFS media. +/// +/// /// -/// Only the tag is verified; the real LIN tree (logical inode number tree) -/// is a cluster-wide construct and cannot be walked from a single image. -/// File data is FEC-striped across nodes (N+M:B protection groups) — even -/// a complete single-drive image carries only one stripe and cannot -/// reconstruct file content without peer nodes. +/// Dell documents the cluster-level structures required to resolve OneFS data: +/// superblocks live at multiple fixed block addresses on every drive and point +/// at the LIN master, the LIN B+ tree maps logical inode numbers to mirrored +/// inode addresses, and file metatrees map logical blocks to protection groups. +/// Those addresses identify a node, drive and physical block, so a lone drive +/// image is not a self-contained filesystem namespace. /// /// -/// OneFS shares OS ancestry with FreeBSD, but the on-disk filesystem layer -/// is proprietary and NOT UFS-compatible: there is no UFS1 superblock magic -/// (0x00011954) at the UFS1 superblock offset (8192). The OneFS -/// on-disk format has never been publicly specified by Dell EMC. +/// Dell also documents enough physical geometry for safe analysis: every data +/// disk is divided into 32 MiB cylinder groups made of 8 KiB filesystem blocks, +/// with a per-cylinder-group bitmap tracking whether blocks are used for data, +/// inodes or other metadata. The bitmap's raw serialization and location are not +/// publicly specified, so this reader reports the geometry but does not attempt +/// to interpret allocation state. /// -/// +/// +/// Dell does not publish a fixed raw-image magic value that identifies a OneFS +/// drive at offset zero. In particular, the historical "OneFS" / "ONEF" +/// literals previously used here could not be corroborated by Dell documentation +/// and are therefore not parsed or advertised as signatures. Public OneFS logs +/// do show that a superblock magic is validated, but neither its authoritative +/// value nor the fixed superblock block addresses have been found in a public +/// byte-level specification. +/// +/// +/// The reader deliberately performs no payload reads while listing. This keeps a +/// multi-terabyte disk image out of managed memory and, more importantly, avoids +/// inventing structure from undocumented bytes. The raw image remains available +/// as a bounded streaming entry for forensic inspection or export. +/// +/// public sealed class OneFsReader : IDisposable { - /// OneFS LIN-tree-root long tag: ASCII "OneFS" (5 bytes). - public static readonly byte[] LongTag = "OneFS"u8.ToArray(); - /// OneFS short tag: ASCII "ONEF" (4 bytes, 0x4F4E4546 BE). - public static readonly byte[] ShortTag = "ONEF"u8.ToArray(); + /// Documented OneFS filesystem block size. + public const int PhysicalBlockSize = 8 * 1024; + + /// Documented physical cylinder-group size of each OneFS data disk. + public const int CylinderGroupSize = 32 * 1024 * 1024; + + /// Documented number of 8 KiB blocks in a 32 MiB cylinder group. + public const int BlocksPerCylinderGroup = CylinderGroupSize / PhysicalBlockSize; - private readonly byte[] _data; + /// Name of the synthetic inspection metadata entry. + public const string MetadataEntryName = "metadata.ini"; + + /// Name of the opaque raw-image entry. + public const string RawImageEntryName = "onefs-volume.bin"; + + private readonly Stream _stream; + private readonly bool _leaveOpen; private readonly List _entries = []; + /// Gets the synthetic entries exposed by the conservative reader. + public IReadOnlyList Entries => this._entries; + /// - /// Gets the entries. - /// - public IReadOnlyList Entries => _entries; - /// - /// Gets or sets the tag. + /// Legacy compatibility property. No authoritative fixed OneFS raw-media tag + /// is currently known, so this value is always empty. /// public string Tag { get; private set; } = ""; + /// - /// Gets or sets the trailing word. + /// Legacy compatibility property. No undocumented trailing header word is + /// interpreted; this value is always zero. /// public uint TrailingWord { get; private set; } + /// - /// Gets a value indicating whether valid header. + /// Legacy compatibility property. The current reader does not claim to have + /// validated a proprietary raw-media header, so this value is always false. /// public bool ValidHeader { get; private set; } + /// Gets the byte length of the opaque source image. + public long ImageSize { get; } + + /// + /// Initializes a conservative OneFS image reader without consuming or owning + /// the source stream. Kept as the original one-argument public constructor for + /// binary/source API compatibility. + /// + public OneFsReader(Stream stream) : this(stream, leaveOpen: true) { } + /// - /// Initializes a new instance of . + /// Initializes a conservative OneFS image reader without consuming the image. /// - public OneFsReader(Stream stream) { - using var ms = new MemoryStream(); - stream.CopyTo(ms); - _data = ms.ToArray(); - Parse(); + /// Readable, seekable raw image stream. + /// Whether disposing the reader leaves the source open. + public OneFsReader(Stream stream, bool leaveOpen) { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanRead) + throw new ArgumentException("OneFS inspection requires a readable stream.", nameof(stream)); + if (!stream.CanSeek) + throw new ArgumentException("OneFS inspection requires a seekable stream.", nameof(stream)); + if (stream.Length <= 0) + throw new InvalidDataException("OneFS: empty image cannot be inspected."); + + this._stream = stream; + this._leaveOpen = leaveOpen; + this.ImageSize = stream.Length; + this.BuildEntries(); } - private void Parse() { - if (_data.Length < 8) - throw new InvalidDataException("OneFS: file too small for LIN-tree-root header."); - - if (_data.AsSpan(0, 5).SequenceEqual(LongTag)) { - this.Tag = "OneFS"; - this.TrailingWord = _data.Length >= 12 - ? BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(8, 4)) : 0; - } else if (_data.AsSpan(0, 4).SequenceEqual(ShortTag)) { - this.Tag = "ONEF"; - this.TrailingWord = BinaryPrimitives.ReadUInt32BigEndian(_data.AsSpan(4, 4)); - } else { - throw new InvalidDataException("OneFS: missing 'OneFS' / 'ONEF' tag at offset 0."); - } - - this.ValidHeader = true; - - var meta = BuildMetadata(); - _entries.Add(new OneFsEntry { Name = "metadata.ini", Size = meta.Length, IsDirectory = false, Offset = 0, Data = meta }); - _entries.Add(new OneFsEntry { Name = "onefs-volume.bin", Size = _data.Length, IsDirectory = false, Offset = 0, Data = _data }); + private void BuildEntries() { + var metadata = this.BuildMetadata(); + this._entries.Add(new OneFsEntry { + Name = MetadataEntryName, + Size = metadata.Length, + IsDirectory = false, + Offset = 0, + Data = metadata, + }); + this._entries.Add(new OneFsEntry { + Name = RawImageEntryName, + Size = this.ImageSize, + IsDirectory = false, + Offset = 0, + Data = [], + }); } private byte[] BuildMetadata() { - var bldr = new StringBuilder(); - bldr.Append("parse_status=detection-only\n"); - bldr.Append("stage=0\n"); - bldr.Append("format=Dell EMC Isilon OneFS LIN-tree root\n"); - bldr.Append(CultureInfo.InvariantCulture, $"magic_tag={this.Tag}\n"); - bldr.Append("magic_offset=0\n"); - bldr.Append(CultureInfo.InvariantCulture, $"trailing_word=0x{this.TrailingWord:X8}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"image_size={_data.Length}\n"); - bldr.Append("ro_promotion=blocked\n"); - bldr.Append("ro_promotion_reason_1=FEC-striped across nodes (N+M:B protection groups); single-image carries one stripe only\n"); - bldr.Append("ro_promotion_reason_2=LIN tree is cluster-wide, no per-image inode-to-block mapping\n"); - bldr.Append("ro_promotion_reason_3=on-disk format is proprietary; Dell EMC has not published a spec\n"); - bldr.Append("ro_promotion_reason_4=FreeBSD-kernel ancestry does NOT imply UFS on-disk compatibility (no UFS1 superblock at offset 8192)\n"); - bldr.Append("note=Stage 0 — detection only. OneFS is a clustered scale-out NAS; "); - bldr.Append("LIN tree is cluster-wide, no single-image content surface.\n"); - return Encoding.UTF8.GetBytes(bldr.ToString()); + var builder = new StringBuilder(); + builder.Append("parse_status=opaque-single-image\n"); + builder.Append("stage=0\n"); + builder.Append("format=Dell PowerScale / Isilon OneFS\n"); + builder.Append("detection=extension-or-explicit-selection\n"); + builder.Append("authoritative_raw_magic=not_published\n"); + builder.Append("superblock_magic=known-to-exist-value-not-publicly-verified\n"); + builder.Append("superblock_locations=multiple-fixed-block-addresses-values-not-publicly-verified\n"); + builder.Append("superblock_role=references-LIN-master\n"); + builder.Append("physical_block_size=").Append(PhysicalBlockSize.ToString(CultureInfo.InvariantCulture)).Append('\n'); + builder.Append("cylinder_group_size=").Append(CylinderGroupSize.ToString(CultureInfo.InvariantCulture)).Append('\n'); + builder.Append("blocks_per_cylinder_group=").Append(BlocksPerCylinderGroup.ToString(CultureInfo.InvariantCulture)).Append('\n'); + builder.Append("allocation_tracking=per-cylinder-group-bitmap-serialization-not-published\n"); + builder.Append("image_size=").Append(this.ImageSize.ToString(CultureInfo.InvariantCulture)).Append('\n'); + builder.Append("rw_promotion=blocked\n"); + builder.Append("rw_promotion_reason_1=OneFS exposes one namespace across the cluster, not one self-contained namespace per drive\n"); + builder.Append("rw_promotion_reason_2=LIN B+ tree entries resolve logical inode numbers to mirrored inode addresses on node+drive+block tuples\n"); + builder.Append("rw_promotion_reason_3=IFM metatrees resolve logical file blocks to protection groups distributed across cluster nodes and drives\n"); + builder.Append("rw_promotion_reason_4=safe writes use distributed two-phase commit and per-node journals\n"); + builder.Append("rw_promotion_reason_5=no published byte-level allocation/tree/journal serialization or offline single-drive checker was found\n"); + builder.Append("maintenance=blocked\n"); + builder.Append("maintenance_reason=bitmap location and encoding, relocation metadata, protection-group membership and transaction rules cannot be proven from one opaque image\n"); + builder.Append("ufs_note=OneFS is FreeBSD-derived and early Isilon material describes BAM as working with or instead of BSD UFS; an isolated /ifs data drive is not established as a generic standalone UFS volume\n"); + builder.Append("note=The raw image is exposed byte-for-byte for inspection only; only documented geometry is interpreted.\n"); + return Encoding.UTF8.GetBytes(builder.ToString()); } /// - /// Decodes the supplied input. + /// Opens an entry as a bounded read-only stream. Opening the raw image resets + /// the source to byte zero but never copies the payload into managed memory. /// - public byte[] Extract(OneFsEntry entry) { + public Stream OpenEntry(OneFsEntry entry) { ArgumentNullException.ThrowIfNull(entry); - return entry.Data; + + if (string.Equals(entry.Name, MetadataEntryName, StringComparison.Ordinal)) + return new BoundedEntryStream(new MemoryStream(entry.Data, writable: false), entry.Data.LongLength, leaveOpen: false); + + if (!string.Equals(entry.Name, RawImageEntryName, StringComparison.Ordinal)) + throw new FileNotFoundException($"OneFS entry not found: {entry.Name}", entry.Name); + + this._stream.Position = 0; + return new BoundedEntryStream(this._stream, this.ImageSize, leaveOpen: true); } /// - /// Releases resources held by this instance. + /// Materializes an entry in memory. Prefer for the raw + /// image so large media remains streaming. /// - public void Dispose() { } + public byte[] Extract(OneFsEntry entry) { + ArgumentNullException.ThrowIfNull(entry); + if (entry.Size > Array.MaxLength) + throw new IOException($"OneFS entry '{entry.Name}' is too large to materialize; use OpenEntry for streaming access."); + + using var source = this.OpenEntry(entry); + using var target = new MemoryStream(entry.Size > 0 ? checked((int)entry.Size) : 0); + source.CopyTo(target); + return target.ToArray(); + } + + /// Releases the source stream when ownership was requested. + public void Dispose() { + if (!this._leaveOpen) + this._stream.Dispose(); + } } diff --git a/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs b/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs index 11a1f2f62..485b70875 100644 --- a/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs +++ b/FileSystems/FileSystem.Refs/RefsBlockRefcountCodec.cs @@ -4,9 +4,10 @@ namespace FileSystem.Refs; /// -/// Exact normal-row codec for root #6 / schema 0xe0b0. The modification stamp -/// at +0x10 and trailing dword at +0x81c are intentionally preserved verbatim; -/// their mutation semantics are not guessed. +/// Exact normal-row codec for root #6 / schema 0xe0b0. Existing-row updates +/// preserve the modification stamp at +0x10 and trailing dword at +0x81c +/// verbatim. Fresh rows require the caller to supply the checkpoint-derived +/// modification stamp explicitly. /// internal static class RefsBlockRefcountCodec { public const int EntriesPerRow = 0x400; @@ -65,6 +66,32 @@ public static byte[] AdjustCounts( return result; } + /// + /// Adds clone references to a sparse Block Refcount row. A completely zero + /// slot is not an unowned cluster: absence from the sparse table represents + /// the ordinary single allocation owner. Its first clone therefore adds that + /// implicit owner before the requested new references. Flagged zero-count + /// slots belong to a separate management class and are not given that baseline. + /// + public static byte[] AddCloneReferences( + ReadOnlySpan original, + IReadOnlyDictionary additionalReferences) { + ArgumentNullException.ThrowIfNull(additionalReferences); + if (!TryGetRange(original, out _, out _) || !HasValidTotal(original)) + throw new InvalidDataException("ReFS Block Refcount row is not a valid normal 0x820-byte row."); + + var deltas = new Dictionary(additionalReferences.Count); + foreach (var (index, additional) in additionalReferences) { + if (additional <= 0) + throw new ArgumentOutOfRangeException(nameof(additionalReferences), "ReFS clone references must be positive."); + var raw = ReadRaw(original, index); + var count = raw & CountMask; + var flags = raw & ~CountMask; + deltas[index] = checked(additional + (count == 0 && flags == 0 ? 1 : 0)); + } + return AdjustCounts(original, deltas); + } + public static bool IsUnflaggedZeroRow(ReadOnlySpan value) { if (!TryGetRange(value, out _, out _) || !HasValidTotal(value)) return false; for (var i = 0; i < EntriesPerRow; ++i) @@ -73,12 +100,24 @@ public static bool IsUnflaggedZeroRow(ReadOnlySpan value) { } public static byte[] BuildKey(ulong startVirtualLcn) { + ValidateRangeStart(startVirtualLcn); var key = new byte[16]; BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(0, 8), startVirtualLcn); BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), EntriesPerRow); return key; } + public static byte[] BuildFreshValue(ulong startVirtualLcn, ulong modificationStamp) { + ValidateRangeStart(startVirtualLcn); + var value = new byte[NormalValueSize]; + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x00, 8), startVirtualLcn); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x08, 8), EntriesPerRow); + BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x10, 8), modificationStamp); + // TotalRefCount and the trailing dword start at zero. Per-cluster words are + // likewise zero until the caller materializes the first shared references. + return value; + } + public static void RefreshTotal(Span value) { if (value.Length < NormalValueSize) throw new InvalidDataException("ReFS Block Refcount value is shorter than 0x820 bytes."); @@ -89,6 +128,14 @@ public static void RefreshTotal(Span value) { BinaryPrimitives.WriteUInt32LittleEndian(value.Slice(0x18, 4), sum); } + private static void ValidateRangeStart(ulong startVirtualLcn) { + if ((startVirtualLcn & ((ulong)EntriesPerRow - 1UL)) != 0) + throw new ArgumentOutOfRangeException( + nameof(startVirtualLcn), + startVirtualLcn, + "ReFS Block Refcount rows must start on a 0x400-cluster boundary."); + } + private static void ValidateIndex(ReadOnlySpan value, int index) { if (value.Length < NormalValueSize) throw new InvalidDataException("ReFS Block Refcount value is shorter than 0x820 bytes."); diff --git a/FileSystems/FileSystem.Refs/RefsBlockRefcountPolicy.cs b/FileSystems/FileSystem.Refs/RefsBlockRefcountPolicy.cs deleted file mode 100644 index 56f55f8da..000000000 --- a/FileSystems/FileSystem.Refs/RefsBlockRefcountPolicy.cs +++ /dev/null @@ -1,188 +0,0 @@ -#pragma warning disable CS1591 -using System.Buffers.Binary; - -namespace FileSystem.Refs; - -/// -/// Functional mutation policy for root #6. It can materialize a previously -/// absent 0x400-cluster row when a singly-owned extent becomes shared, increment -/// existing entries, preserve the two high management flags, and rewrite the -/// TotalRefCount invariant before publishing a CoW replacement root. -/// -internal sealed class RefsBlockRefcountPolicy { - private const ulong RangeClusters = 0x400; - private const int ValueSize = 0x820; - private const int CountsOffset = 0x1C; - private const ushort CountMask = 0x3FFF; - private const ushort FlagsMask = 0xC000; - - private readonly RefsMetadataReader _metadata; - private readonly RefsCowBTree _btree; - - public RefsBlockRefcountPolicy(RefsMetadataReader metadata, RefsCowBTree btree) { - this._metadata = metadata; - this._btree = btree; - } - - /// - /// Adds references to already-live clusters. An untracked cluster is assumed - /// to have its ordinary single live owner; adding one clone therefore creates - /// a refcount of 2 rather than 1. - /// - public RefsCowTreeResult IncrementPhysicalReferences( - IEnumerable physicalClusters, - ushort additionalReferences = 1, - ushort managementFlags = 0) { - if (additionalReferences == 0) throw new ArgumentOutOfRangeException(nameof(additionalReferences)); - if ((managementFlags & ~FlagsMask) != 0) throw new ArgumentOutOfRangeException(nameof(managementFlags)); - var requested = BuildRequest(physicalClusters); - return this.Rewrite(requested, (current, existed) => { - var flags = (ushort)((current & FlagsMask) | managementFlags); - var count = current & CountMask; - var next = existed - ? checked(count + additionalReferences) - : checked(1 + additionalReferences); - if (next > CountMask) throw new OverflowException("ReFS block refcount exceeds its 14-bit on-disk field."); - return (ushort)(flags | next); - }); - } - - public RefsCowTreeResult ClonePhysicalReferences(IEnumerable physicalClusters) - => this.IncrementPhysicalReferences(physicalClusters, additionalReferences: 1); - - public RefsCowTreeResult DecrementPhysicalReferences( - IEnumerable physicalClusters, - ushort references = 1) { - if (references == 0) throw new ArgumentOutOfRangeException(nameof(references)); - var requested = BuildRequest(physicalClusters); - return this.Rewrite(requested, (current, existed) => { - if (!existed) throw new InvalidOperationException("ReFS cannot decrement an untracked shared cluster."); - var flags = (ushort)(current & FlagsMask); - var count = current & CountMask; - if (count < references) - throw new InvalidDataException("ReFS block-refcount underflow would occur."); - return (ushort)(flags | (count - references)); - }); - } - - private RefsCowTreeResult Rewrite( - IReadOnlyDictionary> request, - Func mutate) { - if (this._metadata.Roots.Count <= 6 || this._metadata.Roots[6].Lcns.Count == 0) - throw new NotSupportedException("ReFS Block Refcount root #6 is unavailable."); - - return this._btree.Rewrite(this._metadata.Roots[6], virtualAddresses: true, (rows, comparer) => { - var byStart = new Dictionary(); - for (var i = 0; i < rows.Count; ++i) { - if (!TryParseKey(rows[i].Key, out var start)) continue; - byStart[start] = i; - } - - foreach (var (rangeStart, entries) in request.OrderBy(p => p.Key)) { - RefsTreeRow row; - var rowExists = byStart.TryGetValue(rangeStart, out var rowIndex); - if (rowExists) { - row = rows[rowIndex]; - if (row.Value.Length != ValueSize) - throw new NotSupportedException( - $"ReFS Block Refcount row 0x{rangeStart:X} uses unsupported {row.Value.Length}-byte value form."); - ValidateValue(row.Value, rangeStart); - } else { - var key = BuildKey(rangeStart); - var value = BuildEmptyValue(rangeStart, this._metadata.ActiveCheckpointClock); - row = new RefsTreeRow(key, value, 0); - rowIndex = ~FindInsertion(rows, key, comparer); - if (rowIndex < 0) throw new IOException("ReFS refcount row key unexpectedly already exists."); - rows.Insert(rowIndex, row); - byStart[rangeStart] = rowIndex; - // insertion shifts following indices; rebuild the map once for safety. - byStart.Clear(); - for (var i = 0; i < rows.Count; ++i) - if (TryParseKey(rows[i].Key, out var s)) byStart[s] = i; - rowIndex = byStart[rangeStart]; - } - - var updated = row.Value.ToArray(); - foreach (var (slot, _) in entries) { - var offset = CountsOffset + slot * 2; - var current = BinaryPrimitives.ReadUInt16LittleEndian(updated.AsSpan(offset, 2)); - var existed = rowExists && (current & CountMask) != 0; - var next = mutate(current, existed); - BinaryPrimitives.WriteUInt16LittleEndian(updated.AsSpan(offset, 2), next); - } - BinaryPrimitives.WriteUInt64LittleEndian(updated.AsSpan(0x10, 8), this._metadata.ActiveCheckpointClock + 1); - RefreshTotal(updated); - rows[rowIndex] = row with { Value = updated }; - } - return true; - }); - } - - private IReadOnlyDictionary> BuildRequest(IEnumerable physicalClusters) { - ArgumentNullException.ThrowIfNull(physicalClusters); - var result = new Dictionary>(); - foreach (var physical in physicalClusters.Distinct()) { - if (!this._metadata.TryPhysicalToVirtualLcn(physical, out var vlcn)) - throw new InvalidDataException($"ReFS shared data PLCN 0x{physical:X} has no VLCN mapping."); - var rangeStart = vlcn & ~(RangeClusters - 1); - var index = checked((int)(vlcn - rangeStart)); - if (!result.TryGetValue(rangeStart, out var entries)) result[rangeStart] = entries = []; - entries[index] = physical; - } - if (result.Count == 0) throw new ArgumentException("No ReFS clusters were supplied.", nameof(physicalClusters)); - return result; - } - - private static byte[] BuildKey(ulong start) { - var key = new byte[16]; - BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(0, 8), start); - BinaryPrimitives.WriteUInt64LittleEndian(key.AsSpan(8, 8), RangeClusters); - return key; - } - - private static byte[] BuildEmptyValue(ulong start, ulong stamp) { - var value = new byte[ValueSize]; - BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x00, 8), start); - BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x08, 8), RangeClusters); - BinaryPrimitives.WriteUInt64LittleEndian(value.AsSpan(0x10, 8), stamp); - return value; - } - - private static bool TryParseKey(byte[] key, out ulong start) { - start = 0; - if (key.Length < 16 || BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(8, 8)) != RangeClusters) return false; - start = BinaryPrimitives.ReadUInt64LittleEndian(key.AsSpan(0, 8)); - return true; - } - - private static void ValidateValue(byte[] value, ulong expectedStart) { - if (BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(0x00, 8)) != expectedStart - || BinaryPrimitives.ReadUInt64LittleEndian(value.AsSpan(0x08, 8)) != RangeClusters) - throw new InvalidDataException("ReFS Block Refcount value does not echo its key range."); - uint total = 0; - for (ulong i = 0; i < RangeClusters; ++i) - total = checked(total + (uint)(BinaryPrimitives.ReadUInt16LittleEndian(value.AsSpan(CountsOffset + checked((int)i) * 2, 2)) & CountMask)); - if (BinaryPrimitives.ReadUInt32LittleEndian(value.AsSpan(0x18, 4)) != total) - throw new InvalidDataException("ReFS Block Refcount TotalRefCount does not match its per-cluster array."); - } - - private static void RefreshTotal(Span value) { - uint total = 0; - for (ulong i = 0; i < RangeClusters; ++i) - total = checked(total + (uint)(BinaryPrimitives.ReadUInt16LittleEndian(value.Slice(CountsOffset + checked((int)i) * 2, 2)) & CountMask)); - BinaryPrimitives.WriteUInt32LittleEndian(value.Slice(0x18, 4), total); - } - - private static int FindInsertion(IReadOnlyList rows, byte[] key, RefsKeyComparer comparer) { - var lo = 0; - var hi = rows.Count - 1; - while (lo <= hi) { - var mid = lo + ((hi - lo) >> 1); - var cmp = comparer.Compare(rows[mid].Key, key); - if (cmp == 0) return mid; - if (cmp < 0) lo = mid + 1; - else hi = mid - 1; - } - return ~lo; - } -} diff --git a/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs b/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs index ec54c282d..534be899a 100644 --- a/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs +++ b/FileSystems/FileSystem.Refs/RefsCowBlockRefcountEditor.cs @@ -4,10 +4,14 @@ namespace FileSystem.Refs; /// -/// Native CoW editor for the populated Block Refcount tree. Existing normal -/// rows can be incremented/decremented and are removed when every count/flag -/// reaches zero. Creating a brand-new 0x820 row remains fail-closed because the -/// +0x10 modification-stamp creation rule is not yet proven. +/// Native CoW editor for root #6 / schema 0xe0b0. Existing normal rows can be +/// incremented/decremented, new normal rows are materialized from the active +/// checkpoint clock, and rows are removed when every count/flag reaches zero. +/// +/// The Block Refcount tree is sparse: an untracked live data cluster is treated +/// as having one implicit ordinary owner. Adding the first clone therefore +/// materializes count 2, both when the containing 0x400 row is absent and when +/// the row exists for unrelated shared clusters but the target slot is zero. /// internal sealed class RefsCowBlockRefcountEditor { private readonly RefsMetadataReader _metadata; @@ -51,13 +55,14 @@ public RefsCowTreeResult AdjustPhysicalReferences( return this._tree.Rewrite( this._metadata.Roots[6], virtualAddresses: true, - (rows, _) => ApplyChanges(rows, changes)); + (rows, comparer) => this.ApplyChanges(rows, comparer, changes)); } - private static bool ApplyChanges( + private bool ApplyChanges( List rows, + RefsKeyComparer comparer, IReadOnlyDictionary> changes) { - foreach (var (rowStart, deltas) in changes.OrderBy(item => item.Key)) { + foreach (var (rowStart, requestedDeltas) in changes.OrderBy(item => item.Key)) { var matching = new List(); for (var i = 0; i < rows.Count; ++i) { var candidate = rows[i]; @@ -68,15 +73,22 @@ private static bool ApplyChanges( matching.Add(i); } + if (matching.Count > 1) + throw new InvalidDataException($"ReFS Block Refcount contains duplicate row key 0x{rowStart:X}+0x400."); + if (matching.Count == 0) { - if (deltas.Values.Any(delta => delta < 0)) + if (requestedDeltas.Values.Any(delta => delta < 0)) throw new InvalidDataException( $"ReFS Block Refcount decrement targets untracked VLCN range 0x{rowStart:X}+0x400."); - throw new NotSupportedException( - $"ReFS Block Refcount row creation for VLCN range 0x{rowStart:X}+0x400 requires the unresolved +0x10 creation-stamp rule."); + + var key = RefsBlockRefcountCodec.BuildKey(rowStart); + var fresh = RefsBlockRefcountCodec.BuildFreshValue( + rowStart, + checked(this._metadata.ActiveCheckpointClock + 1)); + var changed = RefsBlockRefcountCodec.AddCloneReferences(fresh, requestedDeltas); + rows.Insert(FindInsertion(rows, key, comparer), new RefsTreeRow(key, changed)); + continue; } - if (matching.Count != 1) - throw new InvalidDataException($"ReFS Block Refcount contains duplicate row key 0x{rowStart:X}+0x400."); var rowIndex = matching[0]; var row = rows[rowIndex]; @@ -87,12 +99,28 @@ private static bool ApplyChanges( throw new NotSupportedException( $"ReFS Block Refcount range 0x{rowStart:X}+0x400 is not a writable normal 0x820-byte row."); - var changed = RefsBlockRefcountCodec.AdjustCounts(row.Value, deltas); - if (RefsBlockRefcountCodec.IsUnflaggedZeroRow(changed)) + var updated = requestedDeltas.Values.All(delta => delta > 0) + ? RefsBlockRefcountCodec.AddCloneReferences(row.Value, requestedDeltas) + : RefsBlockRefcountCodec.AdjustCounts(row.Value, requestedDeltas); + if (RefsBlockRefcountCodec.IsUnflaggedZeroRow(updated)) rows.RemoveAt(rowIndex); else - rows[rowIndex] = row with { Value = changed }; + rows[rowIndex] = row with { Value = updated }; } return true; } + + private static int FindInsertion( + IReadOnlyList rows, + byte[] key, + RefsKeyComparer comparer) { + var lo = 0; + var hi = rows.Count; + while (lo < hi) { + var mid = lo + ((hi - lo) >> 1); + if (comparer.Compare(rows[mid].Key, key) < 0) lo = mid + 1; + else hi = mid; + } + return lo; + } } diff --git a/FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs b/FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs new file mode 100644 index 000000000..622a62cde --- /dev/null +++ b/FileSystems/FileSystem.Refs/RefsOfflineBlockCloner.cs @@ -0,0 +1,263 @@ +#pragma warning disable CS1591 + +namespace FileSystem.Refs; + +internal sealed record RefsOfflineCloneResult( + long LogicalSize, + int SharedClusterCount, + ulong CheckpointPhysicalLcn, + ulong CheckpointClock); + +/// +/// Performs the bounded offline ReFS block-clone operation for two existing, +/// equal-sized ordinary extent-backed files. The destination's stream mapping is +/// replaced by the source mapping through immutable object/Object-Table CoW, +/// while root #6 gains the corresponding shared-cluster reference counts in the +/// same alternate-checkpoint publication. +/// +/// This is deliberately narrower than FSCTL_DUPLICATE_EXTENTS_TO_FILE: it clones +/// a whole cluster-aligned file only, does not create names or extend EOF, and +/// refuses resident, sparse, integrity, already-shared and otherwise undecoded +/// stream layouts. Native mounted-driver/MLog semantics remain fail-closed. +/// +public static class RefsOfflineBlockCloner { + /// + /// Replaces the contents of an existing destination file with a metadata-only + /// clone of an existing source file on an unmounted ReFS image. + /// + public static void CloneWholeFile(Stream image, string sourcePath, string destinationPath) + => CloneWholeFileCore(image, sourcePath, destinationPath); + + internal static RefsOfflineCloneResult CloneWholeFileCore( + Stream image, + string sourcePath, + string destinationPath) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanWrite || !image.CanSeek) + throw new ArgumentException("ReFS offline block clone requires a readable, writable, seekable image.", nameof(image)); + + sourcePath = NormalizePath(sourcePath, nameof(sourcePath)); + destinationPath = NormalizePath(destinationPath, nameof(destinationPath)); + if (sourcePath.Equals(destinationPath, StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException("ReFS block clone source and destination must be different files.", nameof(destinationPath)); + + var metadata = RefsMetadataReader.Open(image); + var files = new RefsNamespaceReader(metadata).ReadAll(); + var source = FindRegularFile(files, sourcePath, nameof(sourcePath)); + var destination = FindRegularFile(files, destinationPath, nameof(destinationPath)); + ValidateWholeFileProfile(metadata, source, destination); + + var sourcePhysical = ExpandPhysicalClusters(metadata, source); + var destinationPhysical = ExpandPhysicalClusters(metadata, destination); + if (sourcePhysical.Count != destinationPhysical.Count) + throw new InvalidDataException("ReFS whole-file clone source and destination have different allocation coverage."); + if (sourcePhysical.Count == 0) + throw new NotSupportedException("ReFS zero-length files do not need block-clone metadata."); + if (sourcePhysical.Distinct().Count() != sourcePhysical.Count) + throw new NotSupportedException("ReFS source file already aliases a physical cluster inside its own extent map."); + if (destinationPhysical.Distinct().Count() != destinationPhysical.Count) + throw new NotSupportedException("ReFS destination file already aliases a physical cluster inside its own extent map."); + if (sourcePhysical.Intersect(destinationPhysical).Any()) + throw new NotSupportedException("ReFS whole-file clone refuses source/destination mappings that already overlap."); + + var refcounts = new RefsBlockRefcount(metadata); + EnsureUnshared(refcounts, sourcePhysical, "source"); + EnsureUnshared(refcounts, destinationPhysical, "destination"); + + var sourceOffsets = sourcePhysical + .Select(lcn => checked((long)lcn * metadata.ClusterSize)) + .ToArray(); + var clonedExtents = RefsStreamLayoutEditor.BuildExtents(metadata, sourceOffsets); + + var store = new RefsCowPageStore(image, metadata); + var tree = new RefsCowBTree(image, metadata, store); + var objectEditor = new RefsCowObjectEditor(metadata, tree); + var objectMutation = objectEditor.UpdateStorageValue( + destinationPath, + location => RefsStreamLayoutEditor.BuildUpdatedValue( + destination, + location.StorageRow, + clonedExtents, + metadata.ClusterSize)); + + var refcountTree = new RefsCowBlockRefcountEditor(metadata, tree) + .IncrementPhysicalReferences(sourcePhysical); + + var checkpoint = PublishOfflineCheckpoint( + image, + metadata, + store, + new Dictionary { + [0] = objectMutation.ObjectTableTree.RootReference, + [6] = refcountTree.RootReference, + }); + + // The clone is already durable at this point. Releasing the destination's + // former ordinary allocation is post-commit reclamation; inability to prove + // allocator ownership leaves a safe leak rather than risking live data. + TryReleaseFormerDestination(image, destinationPhysical); + VerifyClone(image, sourcePath, destinationPath, sourcePhysical); + + return new RefsOfflineCloneResult( + source.Size, + sourcePhysical.Count, + checkpoint.ActiveCheckpointLcn, + checkpoint.ActiveCheckpointClock); + } + + private static RefsMetadataReader PublishOfflineCheckpoint( + Stream image, + RefsMetadataReader metadata, + RefsCowPageStore store, + IReadOnlyDictionary replacements) { + if (!replacements.ContainsKey(0) || !replacements.ContainsKey(6)) + throw new ArgumentException("ReFS block clone must publish Object Table root #0 and Block Refcount root #6 together.", nameof(replacements)); + + var roots = replacements.ToDictionary(item => item.Key, item => item.Value.ToArray()); + var allocatorChanged = false; + foreach (var tier in new[] { + RefsAllocatorTier.Medium, + RefsAllocatorTier.Container, + RefsAllocatorTier.Small, + }) { + if (store.GetReservedClusters(tier).Count == 0) continue; + var publication = new RefsCowAllocatorPublisher(image, metadata, store).Publish(tier); + roots[publication.RootIndex] = publication.Tree.RootReference; + allocatorChanged = true; + } + + var committer = new RefsCheckpointCommitter(image); + var prepared = committer.PrepareNext(); + committer.SetRootReferences(prepared, roots); + committer.Commit(prepared, allocatorChanged: allocatorChanged); + image.Flush(); + + var verify = RefsMetadataReader.Open(image); + foreach (var rootIndex in new[] { 0, 6 }) { + var expected = RefsPageReference.Parse(roots[rootIndex]); + var actual = verify.Roots[rootIndex]; + if (!expected.Lcns.SequenceEqual(actual.Lcns)) + throw new IOException($"ReFS clone checkpoint published but root #{rootIndex} does not match the replacement tree."); + } + return verify; + } + + private static void VerifyClone( + Stream image, + string sourcePath, + string destinationPath, + IReadOnlyList expectedPhysical) { + var metadata = RefsMetadataReader.Open(image); + var files = new RefsNamespaceReader(metadata).ReadAll(); + var source = FindRegularFile(files, sourcePath, nameof(sourcePath)); + var destination = FindRegularFile(files, destinationPath, nameof(destinationPath)); + var sourcePhysical = ExpandPhysicalClusters(metadata, source); + var destinationPhysical = ExpandPhysicalClusters(metadata, destination); + if (!sourcePhysical.SequenceEqual(expectedPhysical) || !destinationPhysical.SequenceEqual(expectedPhysical)) + throw new IOException("ReFS block clone checkpoint is live but source/destination extent mappings do not match."); + + var refcounts = new RefsBlockRefcount(metadata); + foreach (var physical in expectedPhysical) { + if (!refcounts.TryGetPhysical(physical, out var entry) || entry.ReferenceCount != 2) + throw new IOException($"ReFS block clone PLCN 0x{physical:X} does not have the expected reference count 2."); + } + } + + private static void TryReleaseFormerDestination(Stream image, IReadOnlyList physicalClusters) { + try { + var metadata = RefsMetadataReader.Open(image); + var graph = new RefsMetadataGraph(image, metadata); + var remaining = new RefsBlockRefcount(metadata, graph) + .DetachPhysicalReferences(physicalClusters) + .ToHashSet(); + if (remaining.Count == 0) return; + + foreach (var tier in new[] { + RefsAllocatorTier.Medium, + RefsAllocatorTier.Container, + RefsAllocatorTier.Small, + }) { + var writer = new RefsAllocatorWriter(metadata, graph, tier); + var covered = remaining.Where(writer.CoversPhysical).ToArray(); + if (covered.Length == 0) continue; + writer.SetAllocated(covered, allocated: false); + foreach (var lcn in covered) remaining.Remove(lcn); + image.Flush(); + } + // Unknown ownership remains allocated deliberately. + } catch (Exception e) when (e is InvalidDataException or NotSupportedException or IOException or InvalidOperationException) { + // The metadata clone was committed first. Reclamation failure is a safe + // allocation leak and must not turn a durable clone into a false rollback. + } + } + + private static void ValidateWholeFileProfile( + RefsMetadataReader metadata, + RefsFileRecord source, + RefsFileRecord destination) { + if (source.IsResident || destination.IsResident) + throw new NotSupportedException("ReFS whole-file block clone currently requires two extent-backed files."); + if (source.Size != destination.Size) + throw new NotSupportedException("ReFS whole-file block clone requires equal source/destination logical sizes; EOF extension is not implemented."); + if (source.Size <= 0 || source.Size % metadata.ClusterSize != 0) + throw new NotSupportedException("ReFS whole-file block clone currently requires a non-empty cluster-aligned logical size."); + if (source.AllocatedSize != source.Size || destination.AllocatedSize != destination.Size) + throw new NotSupportedException("ReFS whole-file block clone requires fully allocated, non-sparse stream coverage."); + if (source.Extents.Count == 0 || destination.Extents.Count == 0) + throw new InvalidDataException("ReFS extent-backed clone candidate has no decoded data extents."); + if (source.Extents.Any(IsUnsupportedExtent) || destination.Extents.Any(IsUnsupportedExtent)) + throw new NotSupportedException("ReFS whole-file block clone refuses sparse, integrity, snapshot/shared or otherwise special extents."); + } + + private static bool IsUnsupportedExtent(RefsDataExtent extent) + => extent.IsSparse || extent.Flags == 0x1C00D0 || (extent.Flags & 0x04) != 0; + + private static IReadOnlyList ExpandPhysicalClusters( + RefsMetadataReader metadata, + RefsFileRecord file) { + var result = new List(); + uint expectedVcn = 0; + foreach (var extent in file.Extents.OrderBy(item => item.FileVcn)) { + if (extent.FileVcn != expectedVcn) + throw new InvalidDataException($"ReFS file '{file.Path}' extent map is not a contiguous VCN cover."); + for (uint i = 0; i < extent.ClusterCount; ++i) + result.Add(metadata.TranslateVirtualLcn(checked(extent.VirtualLcn + i))); + expectedVcn = checked(expectedVcn + extent.ClusterCount); + } + + var expectedClusters = checked((int)(file.Size / metadata.ClusterSize)); + if (result.Count != expectedClusters) + throw new InvalidDataException( + $"ReFS file '{file.Path}' extent map covers {result.Count} cluster(s), expected {expectedClusters}."); + return result; + } + + private static void EnsureUnshared( + RefsBlockRefcount refcounts, + IEnumerable physicalClusters, + string role) { + foreach (var physical in physicalClusters) + if (refcounts.TryGetPhysical(physical, out var entry) && entry.IsShared) + throw new NotSupportedException( + $"ReFS whole-file clone {role} PLCN 0x{physical:X} is already shared/dedup-managed; repeated clone integration is a separate profile."); + } + + private static RefsFileRecord FindRegularFile( + IEnumerable files, + string path, + string parameterName) { + var file = files.FirstOrDefault(item => string.Equals(item.Path, path, StringComparison.OrdinalIgnoreCase)) + ?? throw new FileNotFoundException($"ReFS file '{path}' was not found.", path); + if (file.IsDirectory) + throw new ArgumentException($"ReFS path '{path}' names a directory.", parameterName); + return file; + } + + private static string NormalizePath(string path, string parameterName) { + ArgumentException.ThrowIfNullOrWhiteSpace(path, parameterName); + var parts = path.Replace('\\', '/').Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 0 || parts.Any(part => part is "." or ".." || part.IndexOf('\0') >= 0)) + throw new ArgumentException("ReFS clone path must be a canonical relative filesystem path.", parameterName); + return string.Join('/', parts); + } +} diff --git a/FileSystems/FileSystem.SysV/SysVFormatDescriptor.cs b/FileSystems/FileSystem.SysV/SysVFormatDescriptor.cs index c9f292655..06a2b4032 100644 --- a/FileSystems/FileSystem.SysV/SysVFormatDescriptor.cs +++ b/FileSystems/FileSystem.SysV/SysVFormatDescriptor.cs @@ -211,7 +211,7 @@ public void Add(Stream archive, IReadOnlyList inputs) { /// /// Removes the named entries from an existing s5fs image via the in-place /// modifier (zeroes file data blocks before returning them to the free - /// list, matching the wipe + /// list, matching the wipe /// contract). Falls back to the rebuild path for any nested-path entry /// the in-place engine won't touch. /// diff --git a/FileSystems/FileSystem.SysV/SysVInPlaceModifier.cs b/FileSystems/FileSystem.SysV/SysVInPlaceModifier.cs index e21a87425..2c4ac71d4 100644 --- a/FileSystems/FileSystem.SysV/SysVInPlaceModifier.cs +++ b/FileSystems/FileSystem.SysV/SysVInPlaceModifier.cs @@ -29,7 +29,7 @@ namespace FileSystem.SysV; /// data block stays byte-identical at its original offset. /// Remove: zero the dirent's inode number in place; wipe each /// data block the inode addresses to all-zero (matching the -/// wipe contract); push the data +/// wipe contract); push the data /// blocks onto s_free[] (spilling the cache to a new chain group /// when the 50th entry would overflow); zero the 64-byte inode slot so /// the re-scan rediscovers it. Every untouched inode and every untouched diff --git a/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs b/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs index 63aaff81d..f8a417f26 100644 --- a/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs +++ b/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs @@ -5,15 +5,23 @@ namespace FileSystem.Tux3; /// -/// Read-only native-superblock descriptor for the linux-tux3 research filesystem. +/// Native-superblock descriptor for the linux-tux3 research filesystem. /// /// /// The descriptor recognises real linux-tux3 disk-format revisions and parses the packed, /// big-endian struct disksuper at byte 4096. Native tree traversal and mutation are not /// implemented, so Create/Modify/Defragment capabilities are intentionally withheld rather than /// routing files through a private side-table that no TUX3 implementation understands. +/// +/// The on-disk volblocks field does, however, define the native volume boundary. +/// Bytes after that boundary are outside the TUX3 volume and can therefore be reported as free, +/// wiped, and removed by shrink without interpreting or modifying any native tree. Everything +/// inside the declared volume remains fail-closed as metadata-reserved until allocation-tree +/// traversal exists. /// -public sealed class Tux3FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, ISyntheticEntryNames { +public sealed class Tux3FormatDescriptor : + IFormatDescriptor, IArchiveFormatOperations, ISyntheticEntryNames, IFilesystemExtentMap, IArchiveShrinkable { + private static readonly HashSet SyntheticNames = new(StringComparer.OrdinalIgnoreCase) { "FULL.tux3", "metadata.ini", "superblock.bin" }; @@ -59,7 +67,8 @@ public sealed class Tux3FormatDescriptor : IFormatDescriptor, IArchiveFormatOper /// public string Description => - "TUX3 version-tree research filesystem — native big-endian superblock detection/metadata; tree traversal and writing not yet implemented."; + "TUX3 version-tree research filesystem — native big-endian superblock detection/metadata, " + + "fail-closed layout mapping, external-padding wipe/shrink; native tree traversal and writing not yet implemented."; /// public List List(Stream stream, string? password) { @@ -81,4 +90,101 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? reader.ExtractTo(entry, output); } } + + /// + /// Enumerates the provable byte layout without guessing at undecoded TUX3 allocation state. + /// The declared native volume is reserved wholesale; only trailing bytes outside it are free. + /// Invalid or arithmetically unrepresentable volume metadata reserves the complete physical image. + /// + public IEnumerable EnumerateExtents(Stream image) { + ArgumentNullException.ThrowIfNull(image); + if (!image.CanRead || !image.CanSeek) + return []; + + var imageLength = image.Length; + try { + using var reader = new Tux3Reader(image); + if (!TryGetDeclaredVolumeLength(reader, out var volumeLength)) + return ReserveWholeImage(imageLength); + + if (volumeLength >= imageLength) + return ReserveWholeImage(imageLength); + + return [ + new DefragBlockInfo(0, volumeLength, DefragBlockKind.MetadataReserved, "TUX3 volume (allocation map unresolved)"), + new DefragBlockInfo(volumeLength, imageLength - volumeLength, DefragBlockKind.Free, "Trailing bytes outside TUX3 volume"), + ]; + } catch (InvalidDataException) { + return ReserveWholeImage(imageLength); + } catch (IOException) { + return ReserveWholeImage(imageLength); + } + } + + /// + /// Removes only bytes beyond the volume size declared by volblocks * blocksize. + /// Malformed, truncated, or arithmetically invalid images are copied through unchanged. + /// + public void Shrink(Stream input, Stream output) { + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(output); + if (!input.CanRead || !input.CanSeek) + throw new ArgumentException("TUX3 shrink requires a readable, seekable input stream.", nameof(input)); + if (!output.CanWrite || !output.CanSeek) + throw new ArgumentException("TUX3 shrink requires a writable, seekable output stream.", nameof(output)); + if (ReferenceEquals(input, output)) + throw new ArgumentException("TUX3 shrink requires distinct input and output streams.", nameof(output)); + + var copyLength = input.Length; + try { + using var reader = new Tux3Reader(input); + if (TryGetDeclaredVolumeLength(reader, out var volumeLength) && volumeLength <= input.Length) + copyLength = volumeLength; + } catch (InvalidDataException) { + // Total operation: malformed input is copied through unchanged. + } catch (IOException) { + // Total operation: an unreadable metadata surface is copied through unchanged. + } + + input.Position = 0; + output.Position = 0; + output.SetLength(0); + CopyPrefix(input, output, copyLength); + } + + private static bool TryGetDeclaredVolumeLength(Tux3Reader reader, out long length) { + length = 0; + // The reference format reserves a 4 KiB superblock area as the maximum block size. + // Treat larger shifts as corrupt rather than trusting them for destructive maintenance. + if (!reader.ValidSuperblock || reader.VolBlocks == 0 || reader.BlockBits > 12) + return false; + + var blockSize = 1UL << reader.BlockBits; + if (reader.VolBlocks > (ulong)long.MaxValue / blockSize) + return false; + + var declared = reader.VolBlocks * blockSize; + if (declared < Tux3Reader.SuperblockOffset + Tux3Reader.DiskSuperSize) + return false; + + length = (long)declared; + return true; + } + + private static IEnumerable ReserveWholeImage(long imageLength) + => imageLength == 0 + ? [] + : [new DefragBlockInfo(0, imageLength, DefragBlockKind.MetadataReserved, "TUX3 image (volume boundary unresolved)")]; + + private static void CopyPrefix(Stream input, Stream output, long count) { + var buffer = new byte[128 * 1024]; + while (count > 0) { + var requested = (int)Math.Min(buffer.Length, count); + var read = input.Read(buffer, 0, requested); + if (read <= 0) + throw new EndOfStreamException("TUX3 input ended before the expected image boundary."); + output.Write(buffer, 0, read); + count -= read; + } + } } diff --git a/FileSystems/FileSystem.Xfs/XfsExtendedAttributes.cs b/FileSystems/FileSystem.Xfs/XfsExtendedAttributes.cs new file mode 100644 index 000000000..2e96a4ffc --- /dev/null +++ b/FileSystems/FileSystem.Xfs/XfsExtendedAttributes.cs @@ -0,0 +1,388 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; +using Compression.Core.Checksums; + +namespace FileSystem.Xfs; + +/// +/// Reads and mutates XFS short-form extended attributes stored in an inode's +/// attribute fork. +/// +/// +/// +/// XFS can promote an attribute fork from short-form to leaf/node btrees and can +/// place large values in remote blocks. This accessor intentionally stops before +/// that boundary: local inode attributes are fully round-trippable, while leaf, +/// node, and remote-value mutation fails closed instead of rewriting structures +/// the current workbench does not yet allocate transactionally. +/// +/// +/// Layout and namespace constants are taken from the published XFS on-disk +/// specification; no Linux implementation code is copied. +/// +/// +public static class XfsExtendedAttributes { + private const uint XfsMagic = 0x58465342; // XFSB + private const ushort InodeMagic = 0x494E; // IN + private const byte FormatLocal = 1; + private const byte FormatExtents = 2; + private const byte FormatBtree = 3; + private const byte AttrRoot = 0x02; + private const byte AttrSecure = 0x04; + private const byte AttrParent = 0x08; + private const byte AttrIncomplete = 0x80; + private const int DiCrcOffset = 100; + private const int ShortFormHeaderSize = 4; // be16 totsize, u8 count, u8 padding + private const int ShortFormEntryHeaderSize = 3; // namelen, valuelen, flags + + /// Reads all short-form xattrs for . + public static IReadOnlyDictionary Read(Stream image, string path) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + EnsureReadableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inode = ReadAt(image, geometry.InodeOffset(inodeNumber), geometry.InodeSize); + return ParseShortForm(inode, geometry).ToDictionary( + attribute => ExpandName(attribute.Flags, attribute.Name), + attribute => attribute.Value, + StringComparer.Ordinal); + } finally { + image.Position = original; + } + } + + /// + /// Creates or replaces a short-form xattr. If the resulting fork does not fit + /// in the inode, the call fails without modifying the image. + /// + public static void Set(Stream image, string path, string name, ReadOnlySpan value) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + EnsureWritableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inodeOffset = geometry.InodeOffset(inodeNumber); + var inode = ReadAt(image, inodeOffset, geometry.InodeSize); + ValidateInode(inode); + + var attributes = ParseShortForm(inode, geometry).ToList(); + var (namespaceFlags, suffix) = CompressName(name); + var suffixBytes = Encoding.UTF8.GetBytes(suffix); + if (suffixBytes.Length is 0 or > byte.MaxValue) + throw new ArgumentException($"XFS xattr name '{name}' is not representable in short-form storage.", nameof(name)); + if (value.Length > byte.MaxValue) + throw new NotSupportedException("XFS short-form xattr values are limited to 255 bytes; remote-value allocation is not implemented."); + + var index = attributes.FindIndex(attribute => + SameNamespace(attribute.Flags, namespaceFlags) && + string.Equals(attribute.Name, suffix, StringComparison.Ordinal)); + if (index >= 0) { + var existing = attributes[index]; + attributes[index] = existing with { Value = value.ToArray() }; + } else { + attributes.Add(new ShortFormAttribute(suffix, value.ToArray(), namespaceFlags)); + } + + WriteShortForm(inode, geometry, attributes); + StampInodeCrcIfNeeded(inode, geometry); + WriteAt(image, inodeOffset, inode); + image.Flush(); + } finally { + image.Position = original; + } + } + + /// Removes one short-form xattr, returning false when it was absent. + public static bool Remove(Stream image, string path, string name) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + EnsureWritableSeekable(image); + + var original = image.Position; + try { + var geometry = Geometry.Read(image); + var inodeNumber = ResolveInode(image, path, geometry.RootInode); + var inodeOffset = geometry.InodeOffset(inodeNumber); + var inode = ReadAt(image, inodeOffset, geometry.InodeSize); + ValidateInode(inode); + + var attributes = ParseShortForm(inode, geometry).ToList(); + var (namespaceFlags, suffix) = CompressName(name); + var removed = attributes.RemoveAll(attribute => + SameNamespace(attribute.Flags, namespaceFlags) && + string.Equals(attribute.Name, suffix, StringComparison.Ordinal)) > 0; + if (!removed) return false; + + WriteShortForm(inode, geometry, attributes); + StampInodeCrcIfNeeded(inode, geometry); + WriteAt(image, inodeOffset, inode); + image.Flush(); + return true; + } finally { + image.Position = original; + } + } + + private static IReadOnlyList ParseShortForm(byte[] inode, Geometry geometry) { + ValidateInode(inode); + var forkoff = inode[82]; + if (forkoff == 0) return []; + + var attrFormat = inode[83]; + if (attrFormat != FormatLocal) + throw new NotSupportedException( + $"XFS: attribute fork format {attrFormat} is not short-form; leaf/btree xattr access is not implemented."); + + var start = checked(geometry.CoreSize + forkoff * 8); + if (start + ShortFormHeaderSize > inode.Length) + throw new InvalidDataException("XFS: short-form attribute fork begins outside the inode."); + + var totalSize = BinaryPrimitives.ReadUInt16BigEndian(inode.AsSpan(start)); + var count = inode[start + 2]; + if (totalSize < ShortFormHeaderSize || start + totalSize > inode.Length) + throw new InvalidDataException("XFS: invalid short-form xattr size."); + + var result = new List(count); + var pos = start + ShortFormHeaderSize; + var end = start + totalSize; + for (var i = 0; i < count; ++i) { + if (pos + ShortFormEntryHeaderSize > end) + throw new InvalidDataException("XFS: truncated short-form xattr entry."); + var nameLength = inode[pos]; + var valueLength = inode[pos + 1]; + var flags = inode[pos + 2]; + var recordLength = ShortFormEntryHeaderSize + nameLength + valueLength; + if (nameLength == 0 || pos + recordLength > end) + throw new InvalidDataException("XFS: malformed short-form xattr entry."); + + var name = Encoding.UTF8.GetString(inode, pos + ShortFormEntryHeaderSize, nameLength); + var value = inode.AsSpan(pos + ShortFormEntryHeaderSize + nameLength, valueLength).ToArray(); + result.Add(new ShortFormAttribute(name, value, flags)); + pos += recordLength; + } + + return result; + } + + private static void WriteShortForm( + byte[] inode, + Geometry geometry, + IReadOnlyList attributes) { + ValidateInode(inode); + + foreach (var attribute in attributes) { + if ((attribute.Flags & (AttrParent | AttrIncomplete)) != 0) + throw new NotSupportedException( + "XFS: parent-pointer or incomplete xattrs cannot be rewritten by the short-form accessor."); + } + + if (attributes.Count == 0) { + var oldForkoff = inode[82]; + if (oldForkoff != 0) { + var oldStart = checked(geometry.CoreSize + oldForkoff * 8); + if (oldStart < inode.Length) inode.AsSpan(oldStart).Clear(); + } + BinaryPrimitives.WriteUInt16BigEndian(inode.AsSpan(80), 0); // di_anextents + inode[82] = 0; // di_forkoff + inode[83] = 0; // di_aformat + return; + } + + if (attributes.Count > byte.MaxValue) + throw new NotSupportedException("XFS short-form xattrs cannot encode more than 255 attributes."); + + var encodedSize = ShortFormHeaderSize; + foreach (var attribute in attributes) { + var nameLength = Encoding.UTF8.GetByteCount(attribute.Name); + if (nameLength is 0 or > byte.MaxValue || attribute.Value.Length > byte.MaxValue) + throw new NotSupportedException("XFS xattr cannot be represented in short-form storage."); + encodedSize = checked(encodedSize + ShortFormEntryHeaderSize + nameLength + attribute.Value.Length); + } + + var existingForkoff = inode[82]; + int forkoff; + if (existingForkoff != 0) { + if (inode[83] != FormatLocal) + throw new NotSupportedException("XFS: existing non-local attribute fork cannot be rewritten as short-form."); + forkoff = existingForkoff; + } else { + var dataBytes = DataForkBytesUsed(inode, geometry); + forkoff = checked((dataBytes + 7) / 8); + if (forkoff == 0) forkoff = 1; + } + + if (forkoff > byte.MaxValue) + throw new NotSupportedException("XFS: data fork leaves no representable attribute-fork offset."); + var start = checked(geometry.CoreSize + forkoff * 8); + if (start + encodedSize > inode.Length) + throw new NotSupportedException( + "XFS: xattrs do not fit in the inode attribute fork; leaf/remote attribute allocation is not implemented."); + + // Build off-image first so a capacity/encoding failure cannot leave a half-written fork. + var encoded = new byte[encodedSize]; + BinaryPrimitives.WriteUInt16BigEndian(encoded, checked((ushort)encodedSize)); + encoded[2] = checked((byte)attributes.Count); + encoded[3] = 0; + var pos = ShortFormHeaderSize; + foreach (var attribute in attributes) { + var nameBytes = Encoding.UTF8.GetBytes(attribute.Name); + encoded[pos] = checked((byte)nameBytes.Length); + encoded[pos + 1] = checked((byte)attribute.Value.Length); + encoded[pos + 2] = attribute.Flags; + nameBytes.AsSpan().CopyTo(encoded.AsSpan(pos + ShortFormEntryHeaderSize)); + attribute.Value.AsSpan().CopyTo(encoded.AsSpan(pos + ShortFormEntryHeaderSize + nameBytes.Length)); + pos += ShortFormEntryHeaderSize + nameBytes.Length + attribute.Value.Length; + } + + // Preserve the data fork and replace only the attribute-fork region. + inode.AsSpan(start).Clear(); + encoded.AsSpan().CopyTo(inode.AsSpan(start)); + BinaryPrimitives.WriteUInt16BigEndian(inode.AsSpan(80), 0); // short-form has no attr extents + inode[82] = checked((byte)forkoff); + inode[83] = FormatLocal; + } + + private static int DataForkBytesUsed(byte[] inode, Geometry geometry) { + var format = inode[5]; + return format switch { + FormatLocal => checked((int)Math.Min( + BinaryPrimitives.ReadUInt64BigEndian(inode.AsSpan(56)), + (ulong)(inode.Length - geometry.CoreSize))), + FormatExtents => checked((int)BinaryPrimitives.ReadUInt32BigEndian(inode.AsSpan(76)) * 16), + FormatBtree => throw new NotSupportedException( + "XFS: cannot introduce an attr fork beside a btree data fork without decoding its in-inode root size."), + _ => 0, + }; + } + + private static string ExpandName(byte flags, string suffix) { + if ((flags & AttrRoot) != 0) return "trusted." + suffix; + if ((flags & AttrSecure) != 0) return "security." + suffix; + if ((flags & AttrParent) != 0) return "xfs.parent." + suffix; + return "user." + suffix; + } + + private static (byte Flags, string Suffix) CompressName(string name) { + if (name.StartsWith("user.", StringComparison.Ordinal)) return (0, name[5..]); + if (name.StartsWith("trusted.", StringComparison.Ordinal)) return (AttrRoot, name[8..]); + if (name.StartsWith("security.", StringComparison.Ordinal)) return (AttrSecure, name[9..]); + throw new ArgumentException( + $"XFS xattr '{name}' has no supported namespace prefix (user., trusted., security.).", + nameof(name)); + } + + private static bool SameNamespace(byte existingFlags, byte requestedFlags) + => (existingFlags & (AttrRoot | AttrSecure | AttrParent)) == requestedFlags; + + private static ulong ResolveInode(Stream image, string path, ulong rootInode) { + var normalized = Normalize(path); + if (normalized.Length == 0) return rootInode; + image.Position = 0; + using var reader = new XfsReader(image, leaveOpen: true); + var entry = reader.Entries.FirstOrDefault(entry => + string.Equals(Normalize(entry.Name), normalized, StringComparison.Ordinal)); + return entry is null + ? throw new FileNotFoundException($"XFS entry not found: {path}") + : checked((ulong)entry.InodeNumber); + } + + private static string Normalize(string path) => path.Replace('\\', '/').Trim('/'); + + private static void ValidateInode(ReadOnlySpan inode) { + if (inode.Length < 100 || BinaryPrimitives.ReadUInt16BigEndian(inode) != InodeMagic) + throw new InvalidDataException("XFS: invalid inode magic."); + } + + private static void StampInodeCrcIfNeeded(byte[] inode, Geometry geometry) { + if (!geometry.IsV5) return; + inode.AsSpan(DiCrcOffset, sizeof(uint)).Clear(); + var crc = Crc32.Compute(inode, Crc32.Castagnoli); + BinaryPrimitives.WriteUInt32LittleEndian(inode.AsSpan(DiCrcOffset), crc); + } + + private static byte[] ReadAt(Stream image, long offset, int length) { + var data = new byte[length]; + image.Position = offset; + image.ReadExactly(data); + return data; + } + + private static void WriteAt(Stream image, long offset, ReadOnlySpan data) { + image.Position = offset; + image.Write(data); + } + + private static void EnsureReadableSeekable(Stream image) { + if (!image.CanRead || !image.CanSeek) + throw new ArgumentException("XFS xattr access requires a readable, seekable image.", nameof(image)); + } + + private static void EnsureWritableSeekable(Stream image) { + EnsureReadableSeekable(image); + if (!image.CanWrite) + throw new ArgumentException("XFS xattr mutation requires a writable image.", nameof(image)); + } + + private sealed record ShortFormAttribute(string Name, byte[] Value, byte Flags); + + private readonly record struct Geometry( + uint BlockSize, + ushort InodeSize, + ulong RootInode, + uint AgBlocks, + byte AgBlockLog, + int CoreSize, + bool IsV5) { + + public static Geometry Read(Stream image) { + if (image.Length < 512) + throw new InvalidDataException("XFS: image too small for its superblock."); + var sb = ReadAt(image, 0, 512); + if (BinaryPrimitives.ReadUInt32BigEndian(sb) != XfsMagic) + throw new InvalidDataException("XFS: invalid superblock magic."); + + var blockSize = BinaryPrimitives.ReadUInt32BigEndian(sb.AsSpan(4)); + var rootInode = BinaryPrimitives.ReadUInt64BigEndian(sb.AsSpan(56)); + var agBlocks = BinaryPrimitives.ReadUInt32BigEndian(sb.AsSpan(84)); + var version = BinaryPrimitives.ReadUInt16BigEndian(sb.AsSpan(100)); + var inodeSize = BinaryPrimitives.ReadUInt16BigEndian(sb.AsSpan(104)); + var agBlockLog = sb[124]; + if (blockSize == 0) blockSize = 4096; + if (inodeSize == 0) inodeSize = 256; + if (agBlocks == 0) agBlocks = checked((uint)(image.Length / blockSize)); + if (agBlockLog == 0) { + var value = agBlocks; + while (value > 1) { ++agBlockLog; value >>= 1; } + } + + var isV5 = (version & 0xF) >= 5; + return new Geometry(blockSize, inodeSize, rootInode, agBlocks, agBlockLog, + CoreSize: isV5 ? 176 : 100, + IsV5: isV5); + } + + public long InodeOffset(ulong inodeNumber) { + var inodesPerBlock = checked((int)(this.BlockSize / this.InodeSize)); + if (inodesPerBlock <= 0 || (inodesPerBlock & (inodesPerBlock - 1)) != 0) + throw new InvalidDataException("XFS: unsupported inode geometry."); + var inoPbLog = 0; + for (var value = inodesPerBlock; value > 1; value >>= 1) ++inoPbLog; + var aginoLog = this.AgBlockLog + inoPbLog; + var agNumber = inodeNumber >> aginoLog; + var agInode = inodeNumber & ((1UL << aginoLog) - 1); + var block = agInode / (ulong)inodesPerBlock; + var slot = agInode % (ulong)inodesPerBlock; + return checked((long)((agNumber * this.AgBlocks + block) * this.BlockSize + slot * this.InodeSize)); + } + } +} diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md index 9b682d0f7..7efd3edb4 100644 --- a/Hawkynt.FileFormats.Archives/README.md +++ b/Hawkynt.FileFormats.Archives/README.md @@ -127,7 +127,7 @@ Every State, Test, Maintenance, Compress/Decompress and Demux/Mux/Remux cell is | [LBR](https://en.wikipedia.org/wiki/LBR_(file_format)) | `Lbr` | `.lbr` | R/W | ✅ | defrag · wipe | | [gaby.de](http://www.gaby.de/cpm/manuals/archive/lbr.txt) | | LhF (LhFloppy) | `LhF` | `.lhf` | R/W | ✅ | defrag · wipe | Whole-disk archiver: entries are track_NNN.raw | [Aminet](https://aminet.net) | | [lrzip](https://en.wikipedia.org/wiki/Rzip#lrzip) | `Lrzip` | `.lrz` | WORM | ✅ | defrag | LZMA-wrapped subtype only; other lrzip subtypes are rejected; single data member | [GitHub](https://github.com/ckolivas/lrzip) | -| Lynx (Commodore) | `Lynx` | `.lnx` | R/W | ✅ | defrag · wipe | Stored entries only | [Archive Team](http://fileformats.archiveteam.org/wiki/Lynx_(Commodore_64)) | +| Lynx (Commodore) | `Lynx` | `.lnx` | R/W | ✅ | defrag · shrink · wipe | Stored entries only | [Archive Team](http://fileformats.archiveteam.org/wiki/Lynx_(Commodore_64)) | | [LHA / LZH](https://en.wikipedia.org/wiki/LHA_(file_format)) | `Lzh` | `.lzh` `.lha` | R/W | ✅ | defrag · wipe | | [GitHub](https://github.com/jca02266/lha) | | [LZX (Amiga)](https://en.wikipedia.org/wiki/LZX) | `LzxAmiga` | `.lzx` | R/W | ✅ | defrag · wipe | | [Aminet](https://aminet.net) | | [NuFX / ShrinkIt](https://en.wikipedia.org/wiki/ShrinkIt) | `NuFx` | `.shk` `.sdk` `.bxy` | R/W | ✅ | defrag · shrink · wipe | | [nulib.com](https://nulib.com/library/FTN.e08002.htm) | diff --git a/Hawkynt.FileFormats.Archives/REFERENCE.md b/Hawkynt.FileFormats.Archives/REFERENCE.md index 27326b100..f7abd6dec 100644 --- a/Hawkynt.FileFormats.Archives/REFERENCE.md +++ b/Hawkynt.FileFormats.Archives/REFERENCE.md @@ -9840,7 +9840,7 @@ Writes a Long Range Zip (lrzip) container with the LZMA subtype. Other methods a Commodore 64 Lynx/LNX archive. The format stores a textual PETSCII-ish directory and uncompressed file extents in 254-byte blocks mirroring a 1541 sector with its two link bytes removed. -Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`, `IFormatOptionsSchema`, `IWipeEmpty`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `IFormatOptionsSchema`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -9862,12 +9862,15 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati | `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Performs the create operation. | | `Defragment` | `void Defragment(Stream archive)` | Lynx data extents are inherently contiguous and ordered by the directory. Defragmentation therefore consists of validating that layout and dropping transport/trailing padding after the last allocated archive block; intrinsic per-block padding is part of the format. | | `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | Performs the defragment operation. | -| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the layout. | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the exact byte layout: live directory text, directory padding, REL side sectors, logical file bytes, per-file block padding, and any trailer beyond the archive allocation. Explicit free extents make generic forensic wiping safe for this format. | | `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | Performs the extract entry to memory operation. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | | `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | | `OpenEntry` | `Stream OpenEntry(Stream archive, string entryName, string password)` | Performs the open entry operation. | +| `Purge` | `void Purge(Stream archive)` | Removes all live entries in one pass while preserving the input archive's BASIC preamble and Lynx signature. The empty directory is emitted at its minimum one-block allocation. | | `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes entries by closing their allocated block range and truncating the shifted tail. REL side-sector blocks are removed together with their data blocks. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Rebuilds the directory at its smallest whole-254-byte allocation and copies the complete existing data area byte-for-byte. This preserves REL side sectors and file-type metadata, while reclaiming directory blocks left behind after removals and any trailing transport data. | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Zeros only byte ranges that the Lynx layout proves unused: directory padding, payload block padding and trailing transport data. Live file bytes and REL side-sector metadata are retained. | ### Namespace `FileFormat.Lz4` @@ -20814,11 +20817,11 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati #### `ZipLayoutMap` -Walks the ZIP central directory and emits the byte-level layout of every local file header, compressed data payload, the central directory itself, and the EOCD record as `DefragBlockInfo` tiles. +Walks a ZIP central directory and emits a fail-closed byte-level layout of local headers, compressed payloads, optional data descriptors, the central directory and EOCD. | Member | Signature | Summary | | --- | --- | --- | -| `Enumerate` | `static IEnumerable Enumerate(Stream archive)` | Enumerates the value. | +| `Enumerate` | `static IEnumerable Enumerate(Stream archive)` | | #### `ZipModifier` diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index d5725e500..954c10597 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -57,24 +57,24 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | Format | Id | State | Compact | Defrag | Wipe | Shrink | Layout | Purge | Proof | Notes | Reference | | --- | --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | --- | --- | --- | -| [Apple IPSW](https://en.wikipedia.org/wiki/IPSW) | `Ipsw` | R/W | — | — | ✅ | — | — | ✅ | own reader + struct-parity tests | ZIP-based firmware archive | [IPSW](https://en.wikipedia.org/wiki/IPSW) | +| [Apple IPSW](https://en.wikipedia.org/wiki/IPSW) | `Ipsw` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | ZIP-based firmware archive | [IPSW](https://en.wikipedia.org/wiki/IPSW) | | [BIN/CUE](https://en.wikipedia.org/wiki/Cue_sheet_(computing)) | `BinCue` | R/W | — | — | ✅ | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [CUE sheet](https://wiki.hydrogenaud.io/index.php?title=Cue_sheet) | | [CDI](https://en.wikipedia.org/wiki/DiscJuggler) | `Cdi` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [DiscJuggler](https://en.wikipedia.org/wiki/DiscJuggler) | -| [Commodore Lynx archive](https://en.wikipedia.org/wiki/Commodore_64) | `Lynx` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 Lynx archive | [Lynx](https://ist.uwaterloo.ca/~schepers/formats/LNX.TXT) | +| [Commodore Lynx archive](https://en.wikipedia.org/wiki/Commodore_64) | `Lynx` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | C64 Lynx archive | [Lynx](https://ist.uwaterloo.ca/~schepers/formats/LNX.TXT) | | [DMG](https://en.wikipedia.org/wiki/Apple_Disk_Image) | `Dmg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | — | [Apple disk images](https://developer.apple.com/library/archive/documentation/DeveloperTools/Conceptual/SoftwareDistribution4/Concepts/sd_disk_images.html) | | [EnCase EWF (E01)](https://en.wikipedia.org/wiki/EnCase#Expert_Witness_File_Format) | `Ewf` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | EnCase E01 segment chain | [libewf documentation](https://github.com/libyal/libewf/tree/main/documentation) | | [Flattened Device Tree Blob](https://en.wikipedia.org/wiki/Devicetree) | `Dtb` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Flattened Device Tree walked as a pseudo-archive | [Devicetree specification](https://www.devicetree.org/specifications/) | -| [Intel HEX](https://en.wikipedia.org/wiki/Intel_HEX) | `IntelHex` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | Intel HEX and Motorola S-record, normalised to `firmware.bin` + metadata; the writer takes one payload and the base and start addresses from `metadata.ini` | [Intel HEX](https://www.keil.com/support/docs/1584/_hlp_hexfile.htm) / [S-record](https://srecord.sourceforge.net/man/man5/srec_motorola.5.html) | +| [Intel HEX](https://en.wikipedia.org/wiki/Intel_HEX) | `IntelHex` | R/W | ✅ | ✅ rebuild | — | — | — | ✅ | own reader + struct-parity tests | Intel HEX and Motorola S-record, normalised to `firmware.bin` + metadata; the writer takes one payload and the base and start addresses from `metadata.ini` | [Intel HEX](https://www.keil.com/support/docs/1584/_hlp_hexfile.htm) / [S-record](https://srecord.sourceforge.net/man/man5/srec_motorola.5.html) | | [MDF/MDS](https://en.wikipedia.org/wiki/Alcohol_120%25) | `Mdf` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Alcohol 120%](https://en.wikipedia.org/wiki/Alcohol_120%25) | | [Nintendo PartitionFS](https://en.wikipedia.org/wiki/Nintendo_Switch) | `Pfs0` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | Nintendo Switch PartitionFS | [PFS0 (switchbrew)](https://switchbrew.org/wiki/NCA#PFS0) | -| [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | -| [PSP CSO/ZSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | `Cso` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Compressed ISO; edits re-pack the affected blocks | [CSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | +| [NRG](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | `Nrg` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | NRG v1/v2 reader; v2 TAO writer; named edits rebuild the embedded ISO | [Nero Burning ROM](https://en.wikipedia.org/wiki/Nero_Burning_ROM) | +| [PSP CSO/ZSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | `Cso` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | Compressed ISO; edits re-pack the affected blocks | [CSO](https://en.wikipedia.org/wiki/CSO_(file_format)) | | [QCOW2](https://en.wikipedia.org/wiki/Qcow) | `Qcow2` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | [`qemu-img check`](https://www.qemu.org/docs/master/tools/qemu-img.html) / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [QCOW2 specification](https://www.qemu.org/docs/master/interop/qcow2.html) | | [T64](https://en.wikipedia.org/wiki/Commodore_64) | `T64` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | C64 tape archive | [T64 (VICE)](https://vice-emu.sourceforge.io/vice_17.html) | | [TAP](https://en.wikipedia.org/wiki/ZX_Spectrum) | `Tap` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | own reader + struct-parity tests | Sinclair / Commodore tape image | [TAP (World of Spectrum)](https://worldofspectrum.net/faq/reference/formats.htm) | -| [TI-TXT (MSP430)](https://en.wikipedia.org/wiki/Texas_Instruments) | `TiTxt` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | TI-TXT firmware dump; the writer takes one payload and the `@address` from `metadata.ini` | [TI-TXT](https://www.ti.com/lit/pdf/slau131) | +| [TI-TXT (MSP430)](https://en.wikipedia.org/wiki/Texas_Instruments) | `TiTxt` | R/W | ✅ | ✅ rebuild | — | — | — | ✅ | own reader + struct-parity tests | TI-TXT sparse firmware dump; edits and canonical rebuild preserve omitted address holes | [TI-TXT](https://www.ti.com/lit/pdf/slau131) | | [U-Boot uImage](https://en.wikipedia.org/wiki/Das_U-Boot) | `UImage` | WORM | — | — | — | — | — | — | own reader + struct-parity tests | U-Boot legacy image header + payload; the writer takes one payload and the header fields from `metadata.ini`, and computes both CRCs | [U-Boot image format](https://docs.u-boot.org/en/latest/usage/fit/index.html) | -| [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | `UefiFv` | R/W | — | — | — | — | — | ✅ | own reader + struct-parity tests | Firmware volume / FFS | [UEFI specification](https://uefi.org/specifications) | +| [UEFI Firmware Volume](https://en.wikipedia.org/wiki/UEFI) | `UefiFv` | R/W | ✅ | ✅ rebuild | ✅ | — | — | ✅ | own reader + struct-parity tests | Firmware volume / FFS | [UEFI specification](https://uefi.org/specifications) | | [VDI](https://en.wikipedia.org/wiki/VirtualBox#Virtual_disk_image) | `Vdi` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [VirtualBox storage](https://www.virtualbox.org/manual/ch05.html) | | [VHD](https://en.wikipedia.org/wiki/VHD_(file_format)) | `Vhd` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Standalone fixed and dynamic; differencing requires a parent chain and is rejected; edits go to the inner filesystem | [Microsoft VHD](https://learn.microsoft.com/windows-server/virtualization/hyper-v/manage/manage-hyper-v-virtual-hard-disks) | | [VHDX](https://en.wikipedia.org/wiki/VHD_(file_format)#VHDX) | `Vhdx` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | — | ✅ | `qemu-img check` / `convert -O raw` / reverse `qemu-img create` | Edits go to the inner filesystem | [MS-VHDX](https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-vhdx/) | @@ -86,7 +86,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | --- | --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | --- | --- | --- | | [DoubleSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DoubleSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` driver under QEMU mounts, reads and writes; real driver CVFs read byte-exact | MSDBL6.0 CVF; stored clusters | [DoubleSpace](https://en.wikipedia.org/wiki/DriveSpace) | | [DriveSpace 3 CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace3` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | `dmsdos` mounts the genuine `GenuineDvr3Writer` output and reads byte-exact | Win95 DVR3 CVF; a Win95 guest oracle and the HiPack codec are still open | [DriveSpace 3 notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/FileSystems/FileSystem.DriveSpace3/FORMAT-NOTES.md) | -| [DriveSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` driver under QEMU mounts, reads and writes; real driver CVFs read byte-exact | MSDSP6.0 CVF; stored clusters | [DriveSpace](https://en.wikipedia.org/wiki/DriveSpace) | +| [DriveSpace CVF](https://en.wikipedia.org/wiki/DriveSpace) | `DriveSpace` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | MS-DOS 6.22 `DRVSPACE` mounts the DoubleSpace / DriveSpace CVFs; real driver CVFs read byte-exact | MSDSP6.0 CVF; stored clusters | [DriveSpace](https://en.wikipedia.org/wiki/DriveSpace) | | [exFAT](https://en.wikipedia.org/wiki/ExFAT) | `ExFat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `exfat` mount r/w + [`fsck.exfat`](https://github.com/exfatprogs/exfatprogs) | — | [exFAT specification](https://learn.microsoft.com/windows/win32/fileio/exfat-specification) | | [FAT Filesystem Image](https://en.wikipedia.org/wiki/File_Allocation_Table) | `Fat` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount r/w + [`fsck.fat`](https://github.com/dosfstools/dosfstools) + [`mtools`](https://www.gnu.org/software/mtools/) + reverse [`mkfs.vfat`](https://github.com/dosfstools/dosfstools) | FAT12/16/32 with long filenames; in-place `FatModifier` | [FAT specification](https://download.microsoft.com/download/1/6/1/161ba512-40e2-4cc9-843a-923143f3456c/fatgen103.doc) | | [FAT+ Filesystem Image (large-file extension)](https://en.wikipedia.org/wiki/File_Allocation_Table#FAT+) | `FatPlus` | R/W | ✅ | ✅ rebuild | ✅ | ✅ | ✅ | ✅ | host-kernel `vfat` mount + [`fsck.fat`](https://github.com/dosfstools/dosfstools) read the written volume | FAT+ large-file extension | [FAT+ draft](https://www.fdos.org/kernel/fatplus.txt) | @@ -112,7 +112,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [ext2/3/4](https://en.wikipedia.org/wiki/Ext4) | `Ext` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`e2fsck -fn`](https://github.com/tytso/e2fsprogs) / `dumpe2fs` / `debugfs` + reverse `mke2fs` | ext2 / ext3 / ext4 | [ext4 documentation](https://www.kernel.org/doc/html/latest/filesystems/ext4/) | | [F2FS](https://en.wikipedia.org/wiki/F2FS) | `F2fs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `f2fs` mount r/w + [`fsck.f2fs`](https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs-tools.git) | — | [F2FS documentation](https://www.kernel.org/doc/html/latest/filesystems/f2fs.html) | | [GFS (Sistina/Red Hat, original)](https://en.wikipedia.org/wiki/GFS2) | `Gfs1` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; the Linux `gfs2` driver rejects GFS1 | Sistina GFS | [GFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | -| [GFS2 (Global File System 2)](https://en.wikipedia.org/wiki/GFS2) | `Gfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.gfs2 -n`](https://pagure.io/gfs2-utils) clean on populated create and after add/replace/remove; reverse `mkfs.gfs2` reader coverage | Standalone `lock_nolock`; stuffed + indirect-tree files; edits rebuild while preserving image-size floor and lock table | [GFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | +| [GFS2 (Global File System 2)](https://en.wikipedia.org/wiki/GFS2) | `Gfs2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`fsck.gfs2 -n`](https://pagure.io/gfs2-utils) + `debugfs.gfs2`; `mkfs.gfs2` image read by the package | Standalone `lock_nolock`; stuffed + indirect-tree files; edits rebuild while preserving image-size floor and lock table | [GFS2](https://www.kernel.org/doc/html/latest/filesystems/gfs2.html) | | [HAMMER (DragonFly BSD)](https://en.wikipedia.org/wiki/HAMMER_(file_system)) | `Hammer` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | DragonFly kernel mount r/w under QEMU + [`hammer show` / `hammer checkmap`](https://www.dragonflybsd.org/hammer/) | Single volume, about 1 GB minimum; edits rebuild the volume | [HAMMER](https://www.dragonflybsd.org/hammer/) | | [HAMMER2 (DragonFly BSD)](https://en.wikipedia.org/wiki/HAMMER2) | `Hammer2` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | DragonFly kernel mount r/w under QEMU | — | [HAMMER2](https://www.dragonflybsd.org/hammer/) | | [HTFS (SCO High Throughput File System)](https://en.wikipedia.org/wiki/High_Throughput_File_System) | `Htfs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests | SCO HTFS | [HTFS](https://en.wikipedia.org/wiki/High_Throughput_File_System) | @@ -129,7 +129,7 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [Reiser4](https://en.wikipedia.org/wiki/Reiser4) | `Reiser4` | WORM | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | own reader + struct-parity tests; `reiser4progs` not installable here | Empty tree from `mkfs.reiser4` captures; files live in a payload area the reiser4 driver does not see, so the writer stays WORM | [Reiser4](https://reiser4.wiki.kernel.org/) | | [ReiserFS](https://en.wikipedia.org/wiki/ReiserFS) | `ReiserFs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | [`reiserfsck`](https://git.kernel.org/pub/scm/linux/kernel/git/jeffm/reiserfsprogs.git), also after mutation | ReiserFS 3.6; edits rebuild the tree | [ReiserFS documentation](https://www.kernel.org/doc/html/latest/filesystems/reiserfs.html) | | [TUX2](https://en.wikipedia.org/wiki/Tux2) | `Tux2` | R | — | — | — | — | — | — | own reader + struct-parity tests | Phase-tree filesystem | [Tux2](https://en.wikipedia.org/wiki/Tux2) | -| [TUX3](https://en.wikipedia.org/wiki/Tux3) | `Tux3` | R | — | — | — | — | — | — | own reader + struct-parity tests | — | [Tux3](https://github.com/OGAWAHirofumi/tux3) | +| [TUX3](https://en.wikipedia.org/wiki/Tux3) | `Tux3` | R | ✅ | — | ✅ | ✅ | — | — | own reader + struct-parity tests | — | [Tux3](https://github.com/OGAWAHirofumi/tux3) | | [UFS](https://en.wikipedia.org/wiki/Unix_File_System) | `Ufs` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | FreeBSD kernel mount r/w + [`fsck_ffs`](https://man.freebsd.org/cgi/man.cgi?query=fsck_ffs) under QEMU | UFS1 / FFS | [UFS](https://en.wikipedia.org/wiki/Unix_File_System) | | [UNIX System V FS](https://en.wikipedia.org/wiki/Unix_File_System) | `SysV` | R/W | ✅ | ✅ moving | ✅ | ✅ | ✅ | ✅ | host-kernel `sysv` mount reads byte-exact | System V filesystem | [sysv](https://www.kernel.org/doc/html/latest/filesystems/sysv-fs.html) | | [VxFS (Veritas)](https://en.wikipedia.org/wiki/Veritas_File_System) | `VxFs` | R/W | ✅ | ✅ moving | ✅ | — | — | ✅ | Linux `freevxfs` driver mounts the written volume | One fileset, direct extents, flat root; edits rebuild the volume | [freevxfs](https://github.com/torvalds/linux/tree/master/fs/freevxfs) | @@ -218,16 +218,16 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | [BeeGFS](https://en.wikipedia.org/wiki/BeeGFS) | `BeeGfs` | R | — | — | — | — | — | — | detection of the on-disk signature | Server-side objects only; no self-contained image exists | [BeeGFS](https://www.beegfs.io/) | | [CephFS / RADOS](https://en.wikipedia.org/wiki/Ceph_(software)) | `CephFs` | R | — | — | — | — | — | — | detection of the on-disk signature | RADOS objects only | [Ceph](https://docs.ceph.com/) | | [Dell EMC Isilon OneFS](https://en.wikipedia.org/wiki/OneFS_distributed_file_system) | `OneFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Isilon OneFS | [OneFS](https://www.dell.com/en-us/dt/storage/powerscale.htm) | -| [eCryptfs](https://en.wikipedia.org/wiki/ECryptfs) | `Ecryptfs` | R | — | — | — | — | — | — | detection of the on-disk signature | Payload is encrypted; plaintext needs the FEK | [eCryptfs](https://www.kernel.org/doc/html/latest/filesystems/ecryptfs.html) | -| [GlusterFS](https://en.wikipedia.org/wiki/Gluster) | `GlusterFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Brick metadata only | [GlusterFS](https://docs.gluster.org/) | +| [eCryptfs](https://en.wikipedia.org/wiki/ECryptfs) | `Ecryptfs` | R/W | ✅ | — | ✅ | ✅ | — | ✅ | [ecryptfs-utils passphrase vector](https://github.com/dustinkirkland/ecryptfs-utils/blob/master/tests/userspace/verify-passphrase-sig.sh) + Linux kernel layout | Passphrase lower-file AES-128/192/256 read/create; private-key auth and xattr-only metadata are unsupported | [eCryptfs](https://www.kernel.org/doc/html/latest/filesystems/ecryptfs.html) | +| [GlusterFS brick](https://en.wikipedia.org/wiki/Gluster) | `GlusterFs` | R | ✅ | — | — | ✅ | — | — | detection of the on-disk signature | Brick metadata only | [GlusterFS](https://docs.gluster.org/) | | [IBM Spectrum Scale / GPFS](https://en.wikipedia.org/wiki/GPFS) | `Gpfs` | R | — | — | — | — | — | — | detection of the on-disk signature | IBM Spectrum Scale | [GPFS](https://www.ibm.com/docs/en/storage-scale) | | [JuiceFS](https://juicefs.com/) | `JuiceFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Chunk objects only | [JuiceFS](https://juicefs.com/docs/community/introduction/) | -| [Lustre](https://en.wikipedia.org/wiki/Lustre_(file_system)) | `Lustre` | R | — | — | — | — | — | — | detection of the on-disk signature | OST / MDT objects only | [Lustre](https://www.lustre.org/) | -| [MooseFS](https://en.wikipedia.org/wiki/Moose_File_System) | `MooseFs` | R | — | — | — | — | — | — | detection of the on-disk signature | Chunk-server objects only | [MooseFS](https://moosefs.com/) | +| [Lustre](https://en.wikipedia.org/wiki/Lustre_(file_system)) | `Lustre` | R | ✅ | — | ✅ | ✅ | — | — | own reader + struct-parity tests | ldiskfs MDT/OST backing store; conservative offline free-block wipe + trailing-block shrink; namespace edits remain R/O | [Lustre](https://www.lustre.org/) | +| [MooseFS](https://en.wikipedia.org/wiki/Moose_File_System) | `MooseFs` | R | — | — | ✅ | — | — | ✅ | detection of the on-disk signature | Chunk-server objects only | [MooseFS](https://moosefs.com/) | | [NetApp WAFL](https://en.wikipedia.org/wiki/Write_Anywhere_File_Layout) | `Wafl` | R | — | — | — | — | — | — | detection of the on-disk signature | NetApp WAFL | [WAFL](https://en.wikipedia.org/wiki/Write_Anywhere_File_Layout) | | [NSS (Novell Storage Services)](https://en.wikipedia.org/wiki/Novell_Storage_Services) | `Nss` | WORM | ✅ | ✅ moving | ✅ | — | — | — | anchor detection derived from real OES media; see the on-disk notes | Beast / B-tree layout undocumented; the writer emits a container under its own magic, not a pool | [NSS on-disk notes](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/NSS-ON-DISK.md) | | [NWFS (Novell NetWare 386 Traditional Filesystem)](https://en.wikipedia.org/wiki/NetWare_File_System) | `Nwfs` | R | — | — | — | — | — | — | detection of the on-disk signature | NetWare 286 / 3.x | [NWFS](https://en.wikipedia.org/wiki/NetWare_File_System) | -| [NWFS386 (Novell NetWare 386 raw)](https://en.wikipedia.org/wiki/NetWare_File_System) | `Nwfs386` | R | — | — | — | — | — | — | detection of the on-disk signature | NetWare 386 raw-partition structures are proprietary | [NWFS](https://en.wikipedia.org/wiki/NetWare_File_System) | +| [NWFS386 (Novell Traditional NetWare filesystem)](https://en.wikipedia.org/wiki/NetWare_File_System) | `Nwfs386` | R/W | ✅ | ✅ rebuild | — | ✅ | ✅ | ✅ | own reader + struct-parity tests | Plain single-segment DOS namespace; rebuild edits preserve block size and volume name; compression/suballocation/alternate namespaces are read-only | [Traditional NetWare partitions](https://www.novell.com/developer/documentation/vfs/vfs__enu/data/afpro44.html) | | [OrangeFS / PVFS2 DBPF](https://en.wikipedia.org/wiki/OrangeFS) | `OrangeFs` | R/W | ✅ | ✅ rebuild | — | — | — | ✅ | own reader + struct-parity tests | One DBPF storage object, not a namespace; the opaque payload is what is edited | [OrangeFS](https://github.com/waltligon/orangefs) | | [SGI CXFS (Cluster XFS)](https://en.wikipedia.org/wiki/CXFS) | `Cxfs` | R | — | — | — | — | — | — | detection of the on-disk signature | SGI cluster XFS | [CXFS](https://en.wikipedia.org/wiki/CXFS) | | [Tahoe-LAFS share](https://en.wikipedia.org/wiki/Tahoe-LAFS) | `TahoeLafs` | R | — | — | — | — | — | — | detection of the on-disk signature | Shares are capability-encrypted; no read-cap, no plaintext | [Tahoe-LAFS](https://tahoe-lafs.org/) | @@ -247,7 +247,6 @@ using var fs = new FatReader(image); foreach (var entry in fs.ListRecursive()) Console.WriteLine($"{entry.Path} {entry.Size,10} {entry.Modified:O}"); ``` - ### Open a virtual disk and inspect its inner filesystem ```csharp @@ -281,7 +280,7 @@ ops.Remove(image, ["var/log/old.log"]); -Every public and protected member of all 833 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.FileSystems/REFERENCE.md). +Every public and protected member of all 836 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.FileSystems/REFERENCE.md). @@ -297,7 +296,6 @@ The **Proof** column is produced by these suites, all in `Compression.Tests`. Ex | `ExternalConformance*`, `ExternalFsInteropTests` (`Category("ExternalFsInterop")`) | `e2fsck`, `xfs_repair`, `btrfs check`, `fsck.f2fs`, `fsck.jfs`, `reiserfsck`, `fsck.minix`, `fsck.hfsplus`, hfsutils, ntfs-3g, `mkudffs`, `unsquashfs`, `mtools`, `qemu-img` accept what the package writes, and the package reads what they write. | | DOS-era drivers under QEMU / `dmsdos` | MS-DOS 6.22 `DRVSPACE` mounts the DoubleSpace / DriveSpace CVFs; the `dmsdos` driver mounts the DriveSpace 3 and Stacker volumes. | | Struct-parity unit tests | For formats with no reachable external tool, each on-disk structure the writer emits is compared field by field against the specification and read back by the package's own reader. | - Disk-image containers follow the same rule with `qemu-img`: forward `check`, raw round-trip via `convert -O raw`, and reverse — an image `qemu-img create` made is opened by the package reader. A forensic-style path builds an inner filesystem with known files, wraps it in a container, validates the container externally, then walks it back and compares bytes. ## 🧯 Filesystem-aware recovery @@ -306,7 +304,7 @@ Disk-image containers follow the same rule with `qemu-img`: forward `check`, raw ```csharp using var fs = File.OpenRead("sdcard.img"); -var hits = new FilesystemCarver().CarveStream(fs); +var hits = FilesystemCarver.CarveStream(fs); foreach (var c in hits) { var result = FilesystemExtractor.ExtractCarved(fs, c, $"out/{c.FormatId}_0x{c.ByteOffset:X}"); Console.WriteLine($"{c.FormatId}: {result.FilesExtracted} files, {result.FilesFailed} failed"); @@ -336,7 +334,7 @@ cwb recover raw.img --mode files --format Jpeg,Png - **Reiser4** remains WORM: the current writer captures the empty native tree but stores added files in a payload area the reiser4 driver does not see. It must not be promoted until the native tree writer owns those files. - **GFS2** R/W is the standalone `lock_nolock` profile with regular files in the root directory; small files are stuffed and larger files use the indirect tree. Existing-image edits rebuild the volume while preserving its size floor and lock-table value. ExHash/nested-directory writing, journal replay, cluster-lock-manager state and extended attributes remain out of scope. - **NSS** is detection only: Novell never published the format, and the writer deliberately emits a container under its own magic that no NetWare or OES release would take for a pool. What was learned from real media is in [NSS-ON-DISK.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/docs/NSS-ON-DISK.md). -- Network, distributed and encrypted formats carry no self-contained image: a single server object, brick or share is recognised and its metadata surfaced, and nothing more is inferred. +- Network and distributed formats generally carry no self-contained image. eCryptfs is the encrypted exception handled here: a lower file is structurally self-contained, and the passphrase profile is supported for AES-128/192/256; private-key authentication and xattr-only metadata remain unsupported. - Disk-image container support and inner-filesystem support are separate capabilities; a container marked R/W edits the filesystem inside it through that filesystem's own descriptor. ## ❤️ Support diff --git a/Hawkynt.FileFormats.FileSystems/REFERENCE.md b/Hawkynt.FileFormats.FileSystems/REFERENCE.md index 99dd39845..9f995e2c2 100644 --- a/Hawkynt.FileFormats.FileSystems/REFERENCE.md +++ b/Hawkynt.FileFormats.FileSystems/REFERENCE.md @@ -196,48 +196,56 @@ Implements `IDisposable`. #### `CsoFormatDescriptor` -PSP CSO / ZSO compressed ISO image. Layout after the 4-byte magic (`CISO` for CSO, `ZISO` for LZ4-compressed ZSO): uint32 header_size, uint64 uncompressed_size, uint32 block_size, uint8 version, uint8 align, uint16 reserved, then an index table of `N = uncompressed_size / block_size + 1` uint32 entries (high bit = stored/uncompressed, low 31 bits = file offset). This descriptor surfaces each compressed block as a raw blob — it does NOT decompress the blocks (consumers can further process with zlib for CSO or LZ4 for ZSO). References: `https://github.com/unknownbrackets/maxcso` — maxcso — maintained CSO/ZSO tool; its docs describe the CSO v1/v2 and ZSO layoutsThe format originates in PSP homebrew (ciso); there is no official Sony documentation +PSP CSO v1/v2 and ZSO compressed ISO image. -Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `ILayoutOptimizable`, `ISyntheticEntryNames`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | | `CsoFormatDescriptor` | `CsoFormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Replaces blocks named `blocks/block_NNNNN.bin` (5-digit zero-padded index) with the supplied payloads. Each input must be exactly the container's block_size bytes. Other input names are ignored. | -| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Emits a fresh CSO v1 stream. Inputs are concatenated in supplied order to form the uncompressed payload (the caller is responsible for ensuring the result is a valid PSP ISO if PSP semantics matter). | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | -| `Remove` | `void Remove(Stream archive, string[] entryNames)` | "Removes" blocks by writing block_size zero bytes through `WriteBlock`, which compresses the zero slab to its minimum DEFLATE encoding and zero-pads the on-disk slack. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `SyntheticEntryNames` | `IReadOnlySet SyntheticEntryNames { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Replaces logical block_NNNNN.bin entries transactionally. | +| `AnalyzeLayout` | `LayoutAnalysis AnalyzeLayout(Stream image)` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Defragment` | `void Defragment(Stream archive)` | Canonical repack: ordered blocks, align=0, no stale/orphaned body bytes. | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Describes the real container byte layout; only bytes after the final index are free. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Purge` | `void Purge(Stream archive)` | Leaves a valid empty container of the same CSO/ZSO variant and block geometry. | +| `RebuildStreaming` | `void RebuildStreaming(Stream source, Stream target, LayoutRebuildOptions options)` | Reblocks and recompresses while preserving the source CSO/ZSO variant and logical ISO bytes. | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | Clears logical blocks to zero; it does not edit the ISO 9660 directory tree inside them. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Rebuilds canonically and uses it only when it is smaller; otherwise copies the source through. | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Wipes dead bytes by canonicalizing into the beginning of the same-sized stream and zeroing the now-unindexed tail. If the local codecs cannot reproduce the image no larger than the source, no byte is touched. | #### `CsoInPlaceModifier` -In-place block-level mutator for PSP CSO v1 images. Lets callers replace individual decompressed blocks without rewriting the whole container. Semantics for `WriteBlock`: If the new compressed payload fits inside the old block's on-disk slot, the payload is written at the same offset and trailing slack is zero-padded — the index table and every other block's bytes are unchanged.Otherwise the payload is appended at the current end of stream and the block's index entry is updated to point at the new location. The old in-place bytes become orphaned (defrag-recoverable).CSO v2 / ZSO (LZ4) are out of scope; only the v1 header layout with align=0 is supported. The modifier refuses to operate on streams whose header reports a different version or a non-zero align (because the offset-shift semantics would silently misplace the new block). +Block-level mutator for CSO v1/v2 and ZSO images. | Member | Signature | Summary | | --- | --- | --- | -| `WriteBlock` | `static void WriteBlock(Stream image, int blockIndex, ReadOnlySpan newUncompressedData)` | Replaces block `blockIndex`'s content with `newUncompressedData` (which must be exactly `block_size` bytes long, matching the container's geometry). The payload is DEFLATE-compressed; if the result is smaller than the slab it's written compressed, otherwise stored uncompressed (with the index entry's high bit set). | +| `WriteBlock` | `static void WriteBlock(Stream image, int blockIndex, ReadOnlySpan newUncompressedData)` | Replaces one logical block. `newUncompressedData` must be exactly block_size bytes; for a partial final block only its logical prefix is retained, matching CSO semantics. | #### `CsoWriter` -Writes a PSP CSO v1 ("CISO") compressed-ISO container from scratch (WORM). Layout (24-byte header + (N+1)·uint32 index + N compressed blocks): $00..$03: magic "CISO"$04..$07: uint32 LE header_size = 24$08..$0F: uint64 LE uncompressed_size$10..$13: uint32 LE block_size (this writer uses 2048 = ISO 9660 sector)$14: uint8 version = 1$15: uint8 align (left-shift applied to index offsets; this writer uses 0)$16..$17: uint16 reserved = 0$18..(header+4·(N+1)): index table. bit 31 set = stored uncompressed.Each block: raw DEFLATE bytes (no zlib header) of one block_size-aligned slab, OR the slab verbatim when the compressed output is not smaller than the slab.ZSO (LZ4) and CSO v2 are out of scope; see `CsoFormatDescriptor`. +Writes PSP CSO v1/v2 and ZSO compressed-ISO containers. | Member | Signature | Summary | | --- | --- | --- | | `CsoWriter` | `CsoWriter()` | | -| `DefaultBlockSize` | `const int DefaultBlockSize` | Default block size — 2048 = one ISO 9660 cooked sector. | -| `Build` | `static byte[] Build(ReadOnlySpan uncompressedData, int blockSize = 2048)` | Builds a CSO v1 stream that, when fully decompressed, yields `uncompressedData`. Blocks of `blockSize` bytes are each DEFLATE-compressed; if the compressed output is not smaller than the original slab, the slab is stored verbatim and its index entry gets the `IndexUncompressedFlag` bit set. | +| `DefaultBlockSize` | `const int DefaultBlockSize` | Default block size — one cooked ISO 9660 sector. | +| `Build` | `static byte[] Build(ReadOnlySpan uncompressedData, int blockSize = 2048)` | Builds a canonical CSO v1 stream around `uncompressedData`. | ### Namespace `FileFormat.Dmg` @@ -535,7 +543,7 @@ Writers for the two firmware text formats, and the shared step that turns a crea | `MetadataName` | `const string MetadataName` | The name the reader gives the rendered summary. | | `PayloadName` | `const string PayloadName` | The name the reader gives the flat payload. | | `ImageFrom` | `static FirmwareImage ImageFrom(IReadOnlyList inputs, string sourceFormat)` | The image a create or edit describes: the single payload input as the bytes, and the addresses read out of `metadata.ini` when the caller passes the one the reader rendered. An input list with no payload describes an image with no data, which both formats can write. | -| `WriteIntelHex` | `static void WriteIntelHex(Stream output, FirmwareImage image, int bytesPerRecord = 16)` | Writes `image` as Intel HEX: type-04 extended-linear-address records whenever the high half of the address changes, type-00 data records of at most `bytesPerRecord` bytes that never straddle a 64 KiB boundary, an optional type-05 start-linear-address record, and the type-01 end-of-file record every reader requires. | +| `WriteIntelHex` | `static void WriteIntelHex(Stream output, FirmwareImage image, int bytesPerRecord = 16)` | Writes `image` as Intel HEX: type-04 extended-linear-address records whenever the high half of the address changes, type-00 data records of at most `bytesPerRecord` bytes that never straddle a 64 KiB boundary, an optional type-03 or type-05 start-address record, and the type-01 end-of-file record every reader requires. | | `WriteTiTxt` | `static void WriteTiTxt(Stream output, FirmwareImage image, int bytesPerLine = 16)` | Writes `image` as TI-TXT: an `@AAAA` address line per segment, space-separated hex bytes at `bytesPerLine` a line, and the single `q` the format ends with. | #### `FirmwareImage` @@ -553,14 +561,15 @@ Implements `IEquatable`. | `Segments` | `IReadOnlyList> Segments { get; init; }` | | | `SourceFormat` | `string SourceFormat { get; init; }` | | | `StartAddress` | `uint? StartAddress { get; init; }` | | +| `StartSegmentAddress` | `ValueTuple? StartSegmentAddress { get; init; }` | Gets the original segmented x86 start address when an Intel HEX type-03 record supplied one. Keeping CS:IP separately matters because many CS:IP pairs map to the same linear address and therefore cannot be reconstructed from `StartAddress` alone. | | `TotalDataBytes` | `int TotalDataBytes { get; init; }` | | | `ToFlatBinary` | `byte[] ToFlatBinary(byte fill = 255)` | Flattens all segments into a single contiguous binary spanning from the lowest address to the end of the highest segment. Gaps are filled with `fill` (default `0xFF` to match flash erase state). | #### `IntelHexFormatDescriptor` -Pseudo-archive descriptor for Intel HEX firmware files. Decodes the ASCII records into a flat binary (`firmware.bin`) and surfaces a `metadata.ini` with record count, declared start address, and gap count. References: Intel "Hexadecimal Object File Format Specification", Rev. A (1988) — the defining document`https://en.wikipedia.org/wiki/Intel_HEX` — record types and checksum rules +Pseudo-archive descriptor for Intel HEX firmware files. Decodes the ASCII records into a flat binary (`firmware.bin`) and surfaces a `metadata.ini` with record count, declared start address, and sparse segment layout. References: Intel "Hexadecimal Object File Format Specification", Rev. A (1988) — the defining document`https://developerhelp.microchip.com/xwiki/bin/view/software-tools/ipe/sqtp-file-format-specification/intel-hex/` — record layout and checksum rules -Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`. | Member | Signature | Summary | | --- | --- | --- | @@ -577,9 +586,10 @@ Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IFormatDescriptor`. | `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | | `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Writes a fresh Intel HEX file: the single payload input becomes the data records, and a `metadata.ini` alongside it -- the one this descriptor's own reader renders -- supplies the base and start addresses that a flat binary cannot carry. | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Writes a fresh Intel HEX file: the single payload input becomes the data records, and a `metadata.ini` alongside it -- the one this descriptor's own reader renders -- supplies the sparse segment map and start-address form that a flat binary cannot carry. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | | `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Purge` | `void Purge(Stream archive)` | Removes all programmed bytes and start-address state while leaving the canonical valid empty Intel HEX document (the EOF record). This overrides the generic pseudo-archive purge because `firmware.bin` is a rendered view that also exists as a zero-length view of an empty image. | #### `IntelHexReader` @@ -601,37 +611,45 @@ Reader for Motorola S-Record files (`Stnn[aaaa|aaaaaa|aaaaaaaa]dd…cc`). Recogn #### `TiTxtFormatDescriptor` -Pseudo-archive descriptor for the TI-TXT firmware text format used by MSP430. Address lines (`@HHHH`) introduce contiguous byte runs; a single `q` terminates the file. Extension is intentionally empty — `.txt` is far too ambiguous — so detection relies on the first non-whitespace byte being `@`. References: Texas Instruments MSP430 programming/bootloader guides — define the TI-TXT format (@addr / data / q)`https://srecord.sourceforge.net` — SRecord tool suite — documents and converts TI-TXT (srec_ti_txt) +Pseudo-archive descriptor for the TI-TXT firmware text format used by MSP430. Address lines (`@HHHH`) introduce sparse byte runs; a line containing `q` terminates the file. The archive view exposes the logical firmware as `firmware.bin` plus a rendered `metadata.ini` summary. References: `https://downloads.ti.com/docs/esd/SPRU513P/Content/SPRU513P_HTML/hex-conversion-utility-description.html` — TI Hex Conversion Utility, TI-TXT format`https://www.ti.com/lit/pdf/slau131` — Texas Instruments MSP430 programming documentation`https://srecord.sourceforge.net/man/man5/srec_ti_txt.5.html` — SRecord TI-TXT documentation; used only as an interoperability reference -Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveWriteConstraints`, `IFormatDescriptor`, `ISyntheticEntryNames`. | Member | Signature | Summary | | --- | --- | --- | | `TiTxtFormatDescriptor` | `TiTxtFormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Writes a fresh TI-TXT file: the single payload input becomes the data lines under an `@address` taken from a `metadata.ini` alongside it, and the file ends with the `q` the format requires. | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `AcceptedInputsDescription` | `string AcceptedInputsDescription { get; }` | | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `MaxTotalArchiveSize` | `long? MaxTotalArchiveSize { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `SyntheticEntryNames` | `IReadOnlySet SyntheticEntryNames { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Replaces the single logical firmware payload and/or moves its base address. A metadata-only base-address edit shifts every sparse section by the same delta so address holes remain holes rather than becoming explicit 0xFF data. | +| `CanAccept` | `bool CanAccept(ArchiveInputInfo input, out string reason)` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Defragment` | `void Defragment(Stream archive)` | Canonicalises section ordering and record wrapping without flattening sparse holes. The reader merges only exactly adjacent sections; the writer then emits one address record per contiguous run and 16 bytes per full data line. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Purge` | `void Purge(Stream archive)` | Erases all live firmware bytes while leaving a valid empty TI-TXT document. | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removing the logical firmware payload leaves the valid empty TI-TXT form, `q`. | #### `TiTxtReader` -Reader for TI-TXT MSP430 text firmware files. Addresses are introduced by `@HHHH` lines; data lines follow as space-separated hex bytes (typically 16 per line); a single `q` token terminates the file. +Parser for Texas Instruments TI-TXT firmware images. Address records introduce sparse byte runs and a line containing only `q` terminates the document. | Member | Signature | Summary | | --- | --- | --- | | `TiTxtReader` | `TiTxtReader()` | | -| `Read` | `static FirmwareImage Read(string text)` | Parses a TI-TXT document into a `FirmwareImage`. | +| `Read` | `static FirmwareImage Read(string text)` | Parses a TI-TXT document into its sparse firmware image. | ### Namespace `FileFormat.Ipsw` @@ -639,31 +657,36 @@ Reader for TI-TXT MSP430 text firmware files. Addresses are introduced by `@HHHH #### `IpswFormatDescriptor` -Apple IPSW / OTA firmware package. An IPSW is just a ZIP file (with an Apple-specific layout). Rather than surfacing entries as a flat generic ZIP, this descriptor lifts the well-known Apple artifacts (`BuildManifest.plist`, `Firmware/` subtree, `LLB.*`, `iBSS.*`, `iBEC.*`, `iBoot.*`, root-filesystem `*.dmg`) into first-class canonical entries. Everything else is exposed under `other/`. This is a compound-extension descriptor (`.ipsw`, `.otazip`): magic is empty so it does not steal generic ZIPs. Read-only; the plist and DMG payloads are emitted as raw bytes — no plist parsing or DMG mounting. References: `https://theapplewiki.com` — The Apple Wiki (formerly The iPhone Wiki) — community IPSW documentation`https://github.com/blacktop/ipsw` — ipsw — maintained IPSW research and extraction tool`https://en.wikipedia.org/wiki/IPSW` — Wikipedia +Apple IPSW / OTA firmware package. IPSW files are ZIP containers whose entry paths are semantically significant: manifests refer to firmware members by their ZIP path, so the descriptor exposes those paths losslessly instead of projecting them into a flattened view. -Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`, `IWipeEmpty`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `ISyntheticEntryNames`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | | `IpswFormatDescriptor` | `IpswFormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | Adds (or replaces by ZIP path) entries inside an existing IPSW. Routes through `IpswInPlaceModifier` — only the central directory, EOCD, and the appended LFH + payload are touched. Synthetic canonical entries are silently dropped. | -| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Emits a fresh IPSW (ZIP) container from the supplied inputs. Synthetic canonical entries the descriptor surfaces on read (`FULL.ipsw`, `metadata.ini`) are silently dropped — they aren't real ZIP entries. All other inputs are stored under their `ArchiveName`. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `SyntheticEntryNames` | `IReadOnlySet SyntheticEntryNames { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Defragment` | `void Defragment(Stream archive)` | | +| `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | | | `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | -| `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes named ZIP entries from an existing IPSW. Routes through `IpswInPlaceModifier` — the LFH + compressed payload of the dropped entry are zero-wiped and the central directory is rewritten. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `OpenEntry` | `Stream OpenEntry(Stream archive, string entryName, string password)` | | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | | +| `Shrink` | `void Shrink(Stream input, Stream output)` | | #### `IpswInPlaceModifier` @@ -682,7 +705,7 @@ In-place modifier for Apple IPSW packages. An IPSW is just a ZIP file with an Ap Commodore 64 Lynx/LNX archive. The format stores a textual PETSCII-ish directory and uncompressed file extents in 254-byte blocks mirroring a 1541 sector with its two link bytes removed. -Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`, `IFormatOptionsSchema`, `IWipeEmpty`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `IFormatOptionsSchema`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -704,12 +727,15 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati | `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Performs the create operation. | | `Defragment` | `void Defragment(Stream archive)` | Lynx data extents are inherently contiguous and ordered by the directory. Defragmentation therefore consists of validating that layout and dropping transport/trailing padding after the last allocated archive block; intrinsic per-block padding is part of the format. | | `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | Performs the defragment operation. | -| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the layout. | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the exact byte layout: live directory text, directory padding, REL side sectors, logical file bytes, per-file block padding, and any trailer beyond the archive allocation. Explicit free extents make generic forensic wiping safe for this format. | | `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | Performs the extract entry to memory operation. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | | `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | | `OpenEntry` | `Stream OpenEntry(Stream archive, string entryName, string password)` | Performs the open entry operation. | +| `Purge` | `void Purge(Stream archive)` | Removes all live entries in one pass while preserving the input archive's BASIC preamble and Lynx signature. The empty directory is emitted at its minimum one-block allocation. | | `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes entries by closing their allocated block range and truncating the shifted tail. REL side-sector blocks are removed together with their data blocks. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Rebuilds the directory at its smallest whole-254-byte allocation and copies the complete existing data area byte-for-byte. This preserves REL side sectors and file-type metadata, while reclaiming directory blocks left behind after removals and any trailing transport data. | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Zeros only byte ranges that the Lynx layout proves unused: directory padding, payload block padding and trailing transport data. Live file bytes and REL side-sector metadata are retained. | ### Namespace `FileFormat.Mdf` @@ -1373,34 +1399,39 @@ Implements `IEquatable
`. #### `UefiFvFormatDescriptor` -UEFI PI Firmware Volume (`.fv`/`.fd`) archive surface. FFS files are exposed as `{GUID}_{TYPE_TAG}.bin`; standalone volumes can be created and ordinary FFS2 records can be added/replaced/removed through erased free space. References: `https://uefi.org/specifications` — UEFI Platform Initialization (PI) Specification, Volume 3: Firmware Storage Design`https://github.com/LongSoft/UEFITool` — UEFITool firmware-volume parser/editor +UEFI PI Firmware Volume archive surface. Standard FFS2/FFS3 files are exposed as `{GUID}_{TYPE_TAG}.bin`; mutable unsigned volumes support transactional edits, erase-aware wiping, purge, and offline consolidation. References: UEFI PI Specification 1.10 Volume III (Firmware Storage) and EDK II as an interoperability oracle. The implementation is independently written. -Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | | `UefiFvFormatDescriptor` | `UefiFvFormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | | `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | | | `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Defragment` | `void Defragment(Stream archive)` | | +| `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Purge` | `void Purge(Stream archive)` | | | `Remove` | `void Remove(Stream archive, string[] entryNames)` | | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Restores unused bytes to the FV's declared erase value. For erase-polarity-one flash this is 0xFF rather than zero; writing zero would turn free space into apparently programmed data and make the firmware volume structurally invalid. | #### `UefiFvReader` -Reader for UEFI Platform Initialization (PI) Firmware Volumes. Locates the FV header by scanning for the `_FVH` signature at offset 40 from the start of each 16-byte-aligned candidate (UEFI PI Volume 3). Walks the FFS file list and returns one `FfsFile` record per live file. +Reader for UEFI PI firmware volumes and standard FFS2/FFS3 file records. | Member | Signature | Summary | | --- | --- | --- | @@ -1447,6 +1478,8 @@ Implements `IEquatable`. | `Attributes` | `uint Attributes { get; init; }` | | | `BlockMap` | `IReadOnlyList> BlockMap { get; init; }` | | | `Checksum` | `ushort Checksum { get; init; }` | | +| `EraseByte` | `byte EraseByte { get; }` | | +| `ErasePolarity` | `bool ErasePolarity { get; }` | | | `ExtHeaderOffset` | `ushort ExtHeaderOffset { get; init; }` | | | `FileSystemGuid` | `Guid FileSystemGuid { get; init; }` | | | `FvLength` | `ulong FvLength { get; init; }` | | @@ -5166,9 +5199,9 @@ Represents an ecryptfs entry. #### `EcryptfsFormatDescriptor` -Read-only descriptor for eCryptfs per-file encryption containers. eCryptfs (Linux) stacks on top of any underlying FS and stores each encrypted file with a 4-byte big-endian marker `0x3C81B7F5` at offset 0 followed by an 8-byte decrypted size, 4-byte flags, and 4-byte extent-size hint. Decryption requires the user's passphrase + EFEK packets — out of scope. The encrypted payload is surfaced as a single opaque entry along with the parsed header metadata. References: `https://docs.kernel.org/filesystems/ecryptfs.html` — Linux kernel eCryptfs documentation`https://github.com/torvalds/linux/tree/master/fs/ecryptfs` — mainline implementation (`ecryptfs_kernel.h` defines the file-header marker + packet layout)`https://en.wikipedia.org/wiki/ECryptfs` — Wikipedia overview +Descriptor for one eCryptfs lower file. eCryptfs is a stacked filesystem: every lower file carries its own encrypted upper-file payload, size, extent geometry and authentication-token packet set. The implementation supports the ordinary passphrase path using the Linux eCryptfs AES-128/192/256 packet conventions. Private-key authentication and xattr-only metadata are deliberately rejected because their required key/xattr material is not contained in a standalone stream. -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -5182,30 +5215,40 @@ Implements `IArchiveFormatOperations`, `IFormatDescriptor`. | `Extensions` | `IReadOnlyList Extensions { get; }` | | | `Family` | `AlgorithmFamily Family { get; }` | | | `Id` | `string Id { get; }` | | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | eCryptfs has no fixed byte magic: its marker is two random 32-bit words whose XOR equals 0x3C81B7F5. Static signature matching cannot express that relation, so detection falls back to the extension and the reader performs the authoritative relational check. | | `Methods` | `IReadOnlyList Methods { get; }` | | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | | `List` | `List List(Stream stream, string password)` | | +| `Purge` | `void Purge(Stream archive)` | Removes the logical payload without requiring the FEK: set plaintext length to zero and truncate all ciphertext extents while retaining valid key metadata. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Drops any bytes after the encrypted extents implied by the header. This is the only meaningful compact/shrink operation for a single eCryptfs lower file. | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Clears metadata padding and bytes beyond the canonical encrypted extent set. Ciphertext inside the final live extent is never touched: plaintext tail zeroing would require the passphrase, which the wipe interface intentionally does not carry. | #### `EcryptfsReader` -Reads eCryptfs per-file encryption headers. eCryptfs is a stacking file-level encryption filesystem (Linux) — every encrypted file is stored on the lower filesystem as a regular file whose first page is a metadata header followed by AES-CBC ciphertext extents. The header starts with a 4-byte big-endian marker (`0x3C81B7F5`) so the individual on-disk container is well-defined and detectable. Decryption requires the user's mount passphrase + EFEK (Encrypted File Encryption Key) tag-3 / tag-11 packets and is OUT OF SCOPE; this reader surfaces the parsed header + the encrypted payload as a single opaque entry. File header layout (big-endian, file offset 0): 0x00 u32 marker == 0x3C81B7F5 0x04 u64 decrypted-size (plaintext length, host-endian on Linux) 0x0C u32 flags 0x10 u32 extent-size (typically 4096) 0x14 ... EFEK packets, tag-3 / tag-11 OpenPGP-style ~0x800 start of AES-CBC ciphertext extents +Reader for an eCryptfs lower file. eCryptfs is a stacked filesystem, so one lower file represents one encrypted upper file rather than a complete volume. Implements `IDisposable`. | Member | Signature | Summary | | --- | --- | --- | -| `EcryptfsReader` | `EcryptfsReader(Stream stream)` | Initializes a new instance of `EcryptfsReader`. | -| `EcryptfsMarker` | `const uint EcryptfsMarker` | Defines the ecryptfs marker constant value. | -| `DecryptedSize` | `ulong DecryptedSize { get; }` | Gets or sets the decrypted size. | -| `Entries` | `IReadOnlyList Entries { get; }` | Gets the entries. | -| `ExtentSize` | `uint ExtentSize { get; }` | Gets or sets the extent size. | -| `Flags` | `uint Flags { get; }` | Gets or sets the flags. | -| `Marker` | `uint Marker { get; }` | Gets or sets the marker. | -| `ValidHeader` | `bool ValidHeader { get; }` | Gets a value indicating whether valid header. | -| `Dispose` | `void Dispose()` | Releases resources held by this instance. | -| `Extract` | `byte[] Extract(EcryptfsEntry entry)` | Decodes the supplied input. | +| `EcryptfsReader` | `EcryptfsReader(Stream stream)` | | +| `CanonicalLength` | `long CanonicalLength { get; }` | Canonical lower-file length implied by the plaintext size and extent geometry. | +| `CipherDescription` | `string CipherDescription { get; }` | Human-readable cipher of the first passphrase packet, when present. | +| `DecryptedSize` | `ulong DecryptedSize { get; }` | Plaintext file length stored in the lower-file header. | +| `Entries` | `IReadOnlyList Entries { get; }` | | +| `ExtentSize` | `uint ExtentSize { get; }` | Encryption extent size in bytes. | +| `FileVersion` | `byte FileVersion { get; }` | eCryptfs lower-file format version. | +| `Flags` | `uint Flags { get; }` | Raw eCryptfs file flags, including the version in the top byte. | +| `HeaderExtentCount` | `ushort HeaderExtentCount { get; }` | Number of header extents preceding ciphertext data. | +| `MarkerWord` | `uint MarkerWord { get; }` | The random first word of the two-word marker pair stored on disk. | +| `Marker` | `uint Marker { get; }` | The validated eCryptfs marker relation constant. | +| `MetadataSize` | `int MetadataSize { get; }` | Total metadata region at the beginning of the lower file. | +| `PassphraseSignature` | `string PassphraseSignature { get; }` | Raw eight-byte authentication-token signature as lowercase hexadecimal. | +| `Dispose` | `void Dispose()` | | +| `ExtractContent` | `byte[] ExtractContent(string password)` | Decrypts the single logical upper-file payload with a passphrase. | +| `ValidatePassword` | `void ValidatePassword(string password)` | Validates a passphrase against the authentication-token packet without materializing plaintext in memory. The ciphertext is streamed through the normal decoder into `Null`, so the same key and extent path used by extraction is exercised before a destructive mutation starts. | ### Namespace `FileSystem.Efs` @@ -5556,7 +5599,7 @@ Builds exFAT filesystem images that Windows 10+ actually mounts. Default layout: ### Namespace `FileSystem.Ext` -[`ExtBlockMover`](#extblockmover) · [`ExtEntry`](#extentry) · [`ExtExtentMap`](#extextentmap) · [`ExtFilesystemDriverAdapter`](#extfilesystemdriveradapter) · [`ExtFormatDescriptor`](#extformatdescriptor) · [`ExtInPlaceShrinker`](#extinplaceshrinker) · [`ExtInPlaceShrinker.ShrinkResult`](#extinplaceshrinkershrinkresult) · [`ExtModifier`](#extmodifier) · [`ExtModifier.InPlaceUnsupportedException`](#extmodifierinplaceunsupportedexception) · [`ExtReader`](#extreader) · [`ExtRemover`](#extremover) · [`ExtShrinkHelper`](#extshrinkhelper) · [`ExtShrinkHelper.ShrinkResult`](#extshrinkhelpershrinkresult) · [`ExtWriter`](#extwriter) · [`ExtWriter.ExtVersion`](#extwriterextversion) +[`ExtBlockMover`](#extblockmover) · [`ExtEntry`](#extentry) · [`ExtExtendedAttributes`](#extextendedattributes) · [`ExtExtentMap`](#extextentmap) · [`ExtFilesystemDriverAdapter`](#extfilesystemdriveradapter) · [`ExtFormatDescriptor`](#extformatdescriptor) · [`ExtInPlaceShrinker`](#extinplaceshrinker) · [`ExtInPlaceShrinker.ShrinkResult`](#extinplaceshrinkershrinkresult) · [`ExtModifier`](#extmodifier) · [`ExtModifier.InPlaceUnsupportedException`](#extmodifierinplaceunsupportedexception) · [`ExtReader`](#extreader) · [`ExtRemover`](#extremover) · [`ExtShrinkHelper`](#extshrinkhelper) · [`ExtShrinkHelper.ShrinkResult`](#extshrinkhelpershrinkresult) · [`ExtWriter`](#extwriter) · [`ExtWriter.ExtVersion`](#extwriterextversion) #### `ExtBlockMover` @@ -5591,6 +5634,16 @@ Represents an ext entry. | `Name` | `string Name { get; init; }` | Gets or sets the name. | | `Size` | `long Size { get; init; }` | Gets or sets the size. | +#### `ExtExtendedAttributes` + +Reads ext2/3/4 extended attributes and performs conservative in-inode mutations. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Read` | `static IReadOnlyDictionary Read(Stream image, string path)` | Reads all directly stored xattrs for `path`. | +| `Remove` | `static bool Remove(Stream image, string path, string name)` | Removes one in-inode xattr, returning false when it was absent. | +| `Set` | `static void Set(Stream image, string path, string name, ReadOnlySpan value)` | Creates or replaces one in-inode xattr. External-block and EA-inode values are never rewritten by this conservative mutator. | + #### `ExtExtentMap` Walks an ext2/3/4 image and yields its actual on-disk byte layout — per-file extent runs (one `DefragBlockInfo` per contiguous block range) plus metadata regions (superblock, group descriptors, block + inode bitmaps, inode table). Used by the defragment window's block-map preview. Streaming: never loads the whole image. All reads flow through a `SectorCache` so multi-TB ext4 images (a 50 TB volume's BGD table + bitmaps are tens of MB) work without OOM. @@ -6817,12 +6870,12 @@ Clean-room GFS2 (Global File System 2) image writer producing a minimal, standal #### `GlusterFsEntry` -Represents a gluster fs entry. +Represents an entry surfaced from a single GlusterFS brick backing store. | Member | Signature | Summary | | --- | --- | --- | | `GlusterFsEntry` | `GlusterFsEntry()` | | -| `Data` | `byte[] Data { get; init; }` | Gets or sets the data. | +| `Data` | `byte[] Data { get; init; }` | Gets or sets eagerly materialized data, used by synthetic metadata entries. | | `IsDirectory` | `bool IsDirectory { get; init; }` | Gets a value indicating whether is directory. | | `Name` | `string Name { get; init; }` | Gets or sets the name. | | `Offset` | `long Offset { get; init; }` | Gets or sets the offset. | @@ -6830,9 +6883,9 @@ Represents a gluster fs entry. #### `GlusterFsFormatDescriptor` -Stage 0 detection-only descriptor for GlusterFS. Honest fallback: GlusterFS has no on-disk image format. A GlusterFS volume is a logical aggregation of one or more "bricks", and every brick is just a normal directory on a local POSIX filesystem (typically XFS or ext4). Volume files live at their normal POSIX paths inside the brick directory and carry GlusterFS state in extended attributes (`trusted.gfid`, `trusted.glusterfs.dht`, `trusted.glusterfs.volume-id`, `trusted.glusterfs.pathinfo`, etc.). There is no superblock, no brick header, no portable single-file representation that this image-based pipeline can consume. We therefore stay Stage 0 permanently. The 0xCAFE5BAB magic recognised here is a workbench-internal convention for hand-dumped brick-object probes — it is not a real on-disk GlusterFS structure and no real GlusterFS deployment will produce it. Promotion to R/O would require walking a live directory tree and reading xattrs, which is outside the image-stream contract enforced by `IArchiveFormatOperations`. References: `https://docs.gluster.org` — official GlusterFS documentation (brick/xattr architecture)`https://github.com/gluster/glusterfs` — canonical source`https://en.wikipedia.org/wiki/GlusterFS` — Wikipedia overview +Read-only single-brick view for GlusterFS backing-store images, with a conservative backing-store shrink operation where it is provably metadata-safe. GlusterFS has no independent block format: a volume is a logical collection of bricks and each brick is an export directory on an ordinary filesystem with extended-attribute support. This descriptor therefore delegates real backing images to the repository's XFS or ext2/3/4 reader and surfaces the physical contents of one brick. It does not reconstruct the distributed Gluster namespace.Detection is extension-only. XFS and ext images already belong to their native descriptors and there is no Gluster-specific superblock magic with which to distinguish a brick image automatically. The former workbench-only 0xCAFE5BAB probe convention is deliberately not recognised as GlusterFS.The native XFS/ext layers can now read Gluster xattrs and conservatively mutate the common short-form/in-inode storage cases. General archive mutation remains disabled until every xattr storage form that a maintenance operation can encounter (including ext external blocks/EA inodes and XFS leaf/btree/remote attributes) is preserved. Advertising a rebuild-based mutation before then could silently discard `trusted.gfid` or `trusted.glusterfs.*` state.Shrink is the deliberate exception. For ext-backed bricks the native in-place shrinker chooses its boundary from the filesystem allocation bitmap; every allocated xattr block therefore pins the boundary just like file data and is preserved byte-for-byte. XFS-backed bricks copy through unchanged because the current XFS shrink path may rebuild the image. This is a backing-store geometry operation only; it is not Gluster's remove-brick command.Gluster volume operations such as rebalance, fix-layout, and remove-brick are explicitly outside this single-image abstraction. They coordinate multiple bricks and belong to a live cluster/volume control plane, not an offline brick image editor. References: `https://docs.gluster.org/en/latest/Administrator-Guide/Setting-Up-Volumes/` — a volume is a logical collection of export-directory bricks`https://docs.gluster.org/en/latest/Administrator-Guide/GlusterFS-Introduction/` — backing filesystems must support extended attributes`https://docs.gluster.org/en/latest/Administrator-Guide/Managing-Volumes/` — remove-brick/rebalance/fix-layout are volume operations`https://docs.kernel.org/filesystems/ext4/attributes.html` — ext4 xattr on-disk layout`https://www.kernel.org/pub/linux/utils/fs/xfs/docs/xfs_filesystem_structure.pdf` — XFS short-form attribute layout`https://github.com/gluster/glusterfs` — canonical GlusterFS implementation, dual GPLv2/LGPLv3+ -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveFormatOperations`, `IArchiveShrinkable`, `IFormatDescriptor`. | Member | Signature | Summary | | --- | --- | --- | @@ -6846,28 +6899,30 @@ Implements `IArchiveFormatOperations`, `IFormatDescriptor`. | `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | | `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | | `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. Empty by design because GlusterFS has no separate on-disk magic. | | `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Extracts entries from the supplied single-brick physical view. | +| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied brick backing-store image. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Shrinks an ext-backed brick to its highest allocated block without rebuilding file or xattr metadata. XFS-backed bricks are copied unchanged. | #### `GlusterFsReader` -Stage 0 detection-only reader for GlusterFS — permanent honest fallback. GlusterFS itself has no on-disk image format: a brick is a normal directory on a local POSIX filesystem (XFS / ext4 / ...) and volume files are stored at their normal POSIX paths inside that directory. All GlusterFS-specific state lives in extended attributes (the `trusted.gfid`, `trusted.glusterfs.dht`, `trusted.glusterfs.volume-id`, `trusted.glusterfs.pathinfo` namespace). Consequences: There is no superblock or brick header to parse.Distribution / replication state (DHT hashing → brick mapping, AFR replicate metadata, EC dispersed metadata, rebalance bookkeeping) only exists across multiple bricks on multiple hosts, not inside any single image.An R/O promotion is fundamentally incompatible with this project's image-stream contract — recognising a GlusterFS "volume" would require walking a live POSIX directory tree and reading xattrs through the host OS, which is outside the `Stream`-based `IArchiveFormatOperations` surface. The 0xCAFE5BAB magic verified by `Parse` is a workbench-internal probe convention used to dump and round-trip hand-crafted "brick object" experiments; it is not a real on-disk GlusterFS marker and no real GlusterFS deployment produces it. The reader therefore stays a thin two-entry detector (synthetic `metadata.ini` + raw `gluster-brick.bin`) and will never grow real semantics. +Reads a single GlusterFS brick backing-store image by delegating the native on-disk filesystem to the existing XFS or ext2/3/4 reader. GlusterFS does not define a separate block format. A brick is an export directory on an ordinary filesystem that supports extended attributes. The logical volume namespace and DHT/AFR/EC state span multiple bricks, while object identity and translator metadata are stored in xattrs such as `trusted.gfid` and `trusted.glusterfs.*`. Consequently this reader intentionally exposes only the physical view of one supplied backing image. It does not claim to reconstruct a Gluster volume from one brick.When the backing namespace contains Gluster's `.glusterfs` GFID index, its parent directory identifies the brick root. Entries outside that subtree and the index itself are omitted from the normal view. If no index is present, the filesystem root is used as a conservative explicit-input fallback.Gluster xattrs are readable through `ReadExtendedAttributes`. The backing accessors also support conservative inline/short-form mutation, but this brick reader remains read-only until all xattr storage forms needed by a maintenance operation can be preserved without rebuilding metadata. Implements `IDisposable`. | Member | Signature | Summary | | --- | --- | --- | -| `GlusterFsReader` | `GlusterFsReader(Stream stream)` | Initializes a new instance of `GlusterFsReader`. | -| `BrickMagic` | `static readonly byte[] BrickMagic` | Workbench-internal probe magic (0xCA 0xFE 0x5B 0xAB, 0xCAFE5BAB big-endian) used by the detector tests. Not a real GlusterFS structure — GlusterFS has no on-disk header at all. | -| `Entries` | `IReadOnlyList Entries { get; }` | Gets the entries. | -| `MagicWord` | `uint MagicWord { get; }` | Gets or sets the magic word. | -| `TrailingWord` | `uint TrailingWord { get; }` | Gets or sets the trailing word. | -| `ValidHeader` | `bool ValidHeader { get; }` | Gets a value indicating whether valid header. | -| `Dispose` | `void Dispose()` | Releases resources held by this instance. | -| `Extract` | `byte[] Extract(GlusterFsEntry entry)` | Decodes the supplied input. | +| `GlusterFsReader` | `GlusterFsReader(Stream stream)` | Initializes a reader over one brick backing-store image. | +| `BackingFileSystem` | `string BackingFileSystem { get; }` | Gets the detected backing filesystem name (`xfs` or `ext`). | +| `BrickRoot` | `string BrickRoot { get; }` | Gets the inferred brick root inside the backing filesystem. Empty means filesystem root. | +| `Entries` | `IReadOnlyList Entries { get; }` | Gets the entries in the single-brick physical view. | +| `HasGlusterIndex` | `bool HasGlusterIndex { get; }` | Gets whether a `.glusterfs` GFID index was present and used to locate the brick root. | +| `ValidHeader` | `bool ValidHeader { get; }` | Gets a value indicating whether a supported backing filesystem was found. | +| `Dispose` | `void Dispose()` | Releases backing filesystem readers and any spool created for a non-seekable input. | +| `Extract` | `byte[] Extract(GlusterFsEntry entry)` | Extracts one surfaced entry. | +| `ReadExtendedAttributes` | `IReadOnlyDictionary ReadExtendedAttributes(GlusterFsEntry entry)` | Reads the native extended attributes belonging to a surfaced brick entry. | ### Namespace `FileSystem.Gpfs` @@ -8651,9 +8706,9 @@ Represents a lustre entry. #### `LustreFormatDescriptor` -R/O descriptor for Lustre MDT/OST images via ldiskfs (ext4-compatible) reader delegation. Surfaces the ldiskfs view of a single MDT or OST backing store — NOT the Lustre logical view (which would require combining MDT inode metadata with file data striped across multiple OSTs, out of scope without live cluster metadata). Detection is extension-routed (.lustre / .ost / .mdt) and the legacy "LUSTRE" / "LUst" object-header magic at offset 0; ext4 superblock magic is deliberately NOT registered here (it would steal detection from generic ext4 images). When opened with an ldiskfs MDT/OST image (recognised by the .ost / .mdt / .lustre extension), `LustreReader` delegates the file walk to `FileSystem.Ext.ExtReader`. References: `https://www.lustre.org/` — project home`https://wiki.lustre.org/` — Lustre wiki (architecture, ldiskfs/MDT/OST layout)`https://en.wikipedia.org/wiki/Lustre_(file_system)` — Wikipedia article +Descriptor for Lustre MDT/OST images backed by ldiskfs (ext4-compatible). The archive projection surfaces one target's backing-store namespace — NOT the distributed Lustre logical view, which requires correlating MDT metadata with objects striped across multiple OSTs. Maintenance is intentionally narrower than generic ext editing. Free-space wipe follows the ldiskfs block allocation bitmap, and shrink updates ext geometry in place, preserving every surviving allocated block. Add/replace/remove, defrag and structural relayout are not advertised because rebuilding through the generic ext writer would discard Lustre LMA/LOV/FID extended attributes and target metadata. Legacy "LUSTRE" / "LUst" object-header dumps remain inspection-only. Detection is extension-routed (.lustre / .ost / .mdt) and the legacy "LUSTRE" / "LUst" object-header magic at offset 0; ext4 superblock magic is deliberately NOT registered here because that would steal generic ext4 images. References: `https://wiki.lustre.org/Configuring_the_Lustre_File_System` — mkfs.lustre examples showing ldiskfs backing targets`https://wiki.lustre.org/Understanding_Lustre_Internals` — backing ldiskfs inspection and Lustre target internals`https://docs.kernel.org/filesystems/ext4/bitmaps.html` — ext4 allocation bitmap semantics and BLOCK_UNINIT warning`https://docs.kernel.org/filesystems/ext4/group_descr.html` — group descriptor and bitmap locations -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveFormatOperations`, `IArchiveShrinkable`, `IFormatDescriptor`, `ILayoutOptimizable`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -8670,8 +8725,11 @@ Implements `IArchiveFormatOperations`, `IFormatDescriptor`. | `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | | `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | +| `AnalyzeLayout` | `LayoutAnalysis AnalyzeLayout(Stream image)` | Reports the current ldiskfs block geometry. Structural relayout is deliberately not offered until a Lustre-aware writer can preserve all target metadata. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | | `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Shrinks an offline ldiskfs target by trimming trailing free blocks while preserving all surviving allocated blocks and Lustre-specific inode/xattr bytes. | +| `WipeUnusedSpace` | `long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true)` | Zeroes complete ldiskfs blocks whose initialized allocation bitmap proves them free. Cluster-tip and deleted-dirent requests are deliberately ignored: those byte ranges cannot be proven dead without interpreting Lustre-specific inode/xattr semantics. | #### `LustreReader` @@ -9320,9 +9378,9 @@ One synthetic surface in the MooseFS master-metadata image: either the `metadata #### `MooseFsFormatDescriptor` -Partial R/O descriptor for MooseFS master-metadata images (`metadata.mfs`). Surfaces the metadata envelope (signature, counters, section index) and the raw payload bytes of each walked section. Path-tree (NODE/EDGE) and chunk-id (CHNK) bodies are version-specific and require golden samples to decode honestly — the reader makes no claim about their internal structure. MooseFS file content lives on chunk servers and is unreachable from a single metadata image. Listing therefore surfaces ONLY synthetic metadata + per-section raw payloads, never POSIX paths. References: `https://github.com/moosefs/moosefs` — canonical source (master metadata dump/load code)`https://moosefs.com/` — vendor site and documentation`https://en.wikipedia.org/wiki/Moose_File_System` — Wikipedia article +Descriptor for MooseFS master-metadata images (`metadata.mfs`). The standalone image contains namespace/chunk metadata, not file payloads; payload bytes live on chunk servers. The descriptor therefore exposes a synthetic forensic view of the metadata envelope rather than pretending a metadata dump is a self-contained MooseFS volume. Safe standalone-image maintenance is deliberately narrow: byte layout can be mapped, unused-space wiping fails closed (valid metadata dumps are tightly packed), and purge resets the image to MooseFS's official `MFSM NEW` empty bootstrap. Add/replace/remove, shrink and defragmentation require semantic NODE/EDGE/CHNK updates and/or live cluster coordination and are not advertised here. References: `https://github.com/moosefs/moosefs` — canonical implementation and metadata loader/store format`https://moosefs.com/` — vendor documentation -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveFormatOperations`, `IArchiveLayoutMap`, `IArchivePurgeable`, `IFormatDescriptor`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -9339,42 +9397,50 @@ Implements `IArchiveFormatOperations`, `IFormatDescriptor`. | `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | | `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `EnumerateLayout` | `IEnumerable EnumerateLayout(Stream archive)` | Enumerates the exact bytes occupied by a valid MooseFS metadata image. Unknown/truncated images return no extents so generic maintenance fails closed. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Extracts the synthetic metadata-image views. | +| `List` | `List List(Stream stream, string password)` | Lists the synthetic entries in the supplied metadata image. | +| `Purge` | `void Purge(Stream archive)` | Resets a valid metadata image to MooseFS's official empty bootstrap state. This only rewrites `metadata.mfs`; any chunk-server data is outside the image and therefore outside this operation. | #### `MooseFsReader` -Partial R/O reader for MooseFS master-metadata images (`metadata.mfs`). MooseFS is a fault-tolerant distributed FS — the master server keeps the namespace + chunk-server topology in a single binary metadata file, while file data lives on chunk servers. This reader understands the master metadata's outer envelope: 8-byte ASCII signature (e.g. `MFSM 2.0`, `MFSM 1.6`, `MFSM 1.5`, `MFSM 1.4`, `MFSM NEW`).For 1.6+ images: two 8-byte big-endian counters (file-id counter, metadata version) immediately after the signature.Sequence of sections. Each section: 8-byte ASCII type tag (`SESS 1.0`, `STAT 1.0`, `NODE 1.0`, `EDGE 1.0`, `FREE 1.0`, `XATR 1.0`, `CHNK 1.0`, `OPEN 1.0`, `FLCK 1.0`, `QUOT 1.0`, `ACLS 1.0`, …) + 8-byte big-endian payload length + that many payload bytes.Final 16-byte terminator `[MFS EOF MARKER]`. The reader walks the section index only — it does not attempt to decode NODE / EDGE record bodies, which differ between MooseFS minor versions and require ground-truth golden samples to validate. NODE/EDGE would give path tree + inode metadata; CHNK gives chunk-id mappings. None of those by themselves yield file content — MooseFS data lives on chunk servers and is only reachable via the live MooseFS protocol. Therefore the reader exposes: `metadata.ini` — human-readable summary of header + section table (name, payload offset, payload length).`moosefs-master.bin` — the raw image, byte-for-byte.`section_.bin` — the raw payload bytes of each section the index walk surfaced (NODE, EDGE, CHNK, …). Useful for offline forensics; we make no claim about their internal structure. If section-walk fails (signature past the 8-byte tag is not recognised, a section length runs past EOF, the EOF marker is missing, …), the reader falls back to a header-only surface (metadata.ini + raw) and records the parse failure in `metadata.ini`'s `parse_status` field. This is the honest "we recognise the envelope but couldn't walk the contents" mode rather than silently inventing entries. +Partial reader for MooseFS master-metadata images (`metadata.mfs`). It understands the versioned outer envelope and section framing, but does not decode the version-specific NODE / EDGE / CHNK bodies or contact chunk servers. Consequently all surfaced archive entries are synthetic forensic views of the metadata image rather than the mounted MooseFS namespace. Implements `IDisposable`. | Member | Signature | Summary | | --- | --- | --- | -| `MooseFsReader` | `MooseFsReader(Stream stream)` | Initializes a new instance of `MooseFsReader`. | -| `EofMarker` | `static readonly byte[] EofMarker` | 16-byte MooseFS end-of-file marker following the last section. | -| `MasterTag` | `static readonly byte[] MasterTag` | MooseFS master metadata 4-byte prefix: ASCII "MFSM". | -| `Entries` | `IReadOnlyList Entries { get; }` | Listing of every entry this image surfaces. | -| `FileIdCounter` | `ulong? FileIdCounter { get; }` | File-id counter from the modern (1.6+) post-signature header, or `null` when the image is too short or pre-1.6. | -| `MetadataVersion` | `ulong? MetadataVersion { get; }` | Metadata version counter from the modern (1.6+) post-signature header, or `null` when the image is too short or pre-1.6. | -| `ParseStatus` | `string ParseStatus { get; }` | Human-readable description of how the section walk terminated: `"ok"` (full walk + EOF marker), `"truncated"` (section walk stopped before EOF marker), `"header-only"` (image too short for any sections), or `"unsupported-header"` (no MFSM tag). | -| `Sections` | `IReadOnlyList Sections { get; }` | Section index walked from the master metadata stream. | -| `Signature` | `string Signature { get; }` | The 8-byte ASCII signature at offset 0 (e.g. `"MFSM 2.0"`). | -| `ValidHeader` | `bool ValidHeader { get; }` | True when the 4-byte `MFSM` tag was present at offset 0. | +| `MooseFsReader` | `MooseFsReader(Stream stream)` | Initializes a reader over one MooseFS metadata image. | +| `EofMarker` | `static readonly byte[] EofMarker` | Modern (1.6+) 16-byte MooseFS end-of-file marker. | +| `MasterTag` | `static readonly byte[] MasterTag` | MooseFS master metadata 4-byte prefix: ASCII `MFSM`. | +| `Entries` | `IReadOnlyList Entries { get; }` | Listing of every synthetic entry surfaced from this image. | +| `FileFormatVersion` | `byte? FileFormatVersion { get; }` | Packed file-format version (`0x16` for 1.6, `0x20` for 2.0), or `null` for the special `MFSM NEW` bootstrap image. | +| `FileIdCounter` | `ulong? FileIdCounter { get; }` | Compatibility view of the first eight bytes after the signature, matching the value older CompressionWorkbench builds exposed under this incorrect name. MooseFS does not define this field as a file-id counter. | +| `ImageSize` | `long ImageSize { get; }` | The exact image size consumed by this reader. | +| `IsEmptyBootstrap` | `bool IsEmptyBootstrap { get; }` | Whether this is MooseFS's official eight-byte empty bootstrap image. | +| `MaxNodeId` | `uint? MaxNodeId { get; }` | Maximum node id from the pre-2.0 metadata header. Not present in 2.0+. | +| `MetaId` | `ulong? MetaId { get; }` | Metadata instance id stored by 2.0+ images. | +| `MetadataVersion` | `ulong? MetadataVersion { get; }` | Metadata/changelog version stored in the master metadata header. | +| `NextSessionId` | `uint? NextSessionId { get; }` | Next session id from the pre-2.0 metadata header. Not present in 2.0+. | +| `ParseStatus` | `string ParseStatus { get; }` | Human-readable parse result: `ok`, `header-only`, `truncated`, `trailing-data`, or `unsupported-header`. | +| `Sections` | `IReadOnlyList Sections { get; }` | Section index walked from section-framed (1.6+) metadata. | +| `Signature` | `string Signature { get; }` | The 8-byte ASCII signature, for example `MFSM 2.0`. | +| `ValidHeader` | `bool ValidHeader { get; }` | True once the `MFSM` prefix has been verified. | | `Dispose` | `void Dispose()` | Releases resources held by this instance. | -| `Extract` | `byte[] Extract(MooseFsEntry entry)` | Returns the bytes that back the given entry (in-memory). | +| `Extract` | `byte[] Extract(MooseFsEntry entry)` | Returns the bytes backing a surfaced synthetic entry. | #### `MooseFsReader.SectionEntry` -One walked section from the master metadata stream. +One walked section from a 1.6+ master metadata stream. Implements `IEquatable`. | Member | Signature | Summary | | --- | --- | --- | -| `SectionEntry` | `SectionEntry(string Tag, long Offset, long Length)` | One walked section from the master metadata stream. | -| `Length` | `long Length { get; init; }` | Length of the section payload in bytes. | -| `Offset` | `long Offset { get; init; }` | Byte offset of the section payload (after the 16-byte tag+length). | -| `Tag` | `string Tag { get; init; }` | The 8-byte ASCII tag (e.g. `"NODE 1.0"`), trimmed. | +| `SectionEntry` | `SectionEntry(string Tag, long Offset, long Length)` | One walked section from a 1.6+ master metadata stream. | +| `Length` | `long Length { get; init; }` | Payload length in bytes. | +| `Offset` | `long Offset { get; init; }` | Byte offset of the section payload. | +| `Tag` | `string Tag { get; init; }` | Eight-byte section tag, for example `NODE 1.0`. | ### Namespace `FileSystem.Msa` @@ -10174,27 +10240,37 @@ Writes a NetWare 386 disk image: a partition table naming one NetWare partition, #### `Nwfs386FormatDescriptor` -Read-only descriptor for Novell NetWare 386 (NWFS386) raw partition dumps, detected via the "NetW" ASCII prefix at offset 0. DOS partition type `0x65`. The on-disk format is FAT-like but proprietary; no parser is attempted — the image is surfaced as a single opaque entry with metadata.ini noting the partition-type hint. References: `https://www.win.tue.nl/~aeb/partitions/partition_types-1.html` — partition-type catalogue (0x65 = Novell NetWare)`https://en.wikipedia.org/wiki/NetWare_File_System` — Wikipedia articleNovell NetWare 386 internal documentation — the on-disk format was never published +Compatibility descriptor for the Novell NetWare 386 Traditional File System. The actual reader/writer lives in `FileSystem.Nwfs`; this descriptor keeps the historical `Nwfs386` id/extensions while sharing that implementation. -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveInMemoryExtract`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveShrinkable`, `IFormatDescriptor`, `IFormatOptionsSchema`, `ILayoutOptimizable`. | Member | Signature | Summary | | --- | --- | --- | | `Nwfs386FormatDescriptor` | `Nwfs386FormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `OptionsSchema` | `IReadOnlyList OptionsSchema { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | | +| `AnalyzeLayout` | `LayoutAnalysis AnalyzeLayout(Stream image)` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `Defragment` | `void Defragment(Stream archive)` | | +| `ExtractEntry` | `void ExtractEntry(Stream input, string entryName, Stream output, string password)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `OpenEntry` | `Stream OpenEntry(Stream archive, string entryName, string password)` | | +| `RebuildStreaming` | `void RebuildStreaming(Stream source, Stream target, LayoutRebuildOptions options)` | | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | | +| `Shrink` | `void Shrink(Stream input, Stream output)` | | ### Namespace `FileSystem.Ocfs2` @@ -10410,45 +10486,52 @@ Represents an one fs entry. #### `OneFsFormatDescriptor` -Stage 0 detection-only descriptor for Dell EMC Isilon OneFS LIN-tree root images. Surfaces only a synthetic `metadata.ini` and the raw image bytes; no real file-walk is attempted. Why R/O promotion is impossible (per CONTRIBUTING.md promotion gates):No single-image content surface. OneFS is a clustered scale-out NAS — every file is split into "protection groups" striped across drives and nodes with FEC (Forward Error Correction, N+M:B layout, e.g. N+2:1). A single drive/node image carries only one stripe; the file data cannot be reconstructed without the peer nodes. A read-only reader from one image can never return correct file bytes. LIN tree is cluster-wide. The Logical Inode Number tree (the OneFS metadata index) lives across nodes, not in a single superblock. There is no per-image inode-to-block mapping to walk. Proprietary on-disk format, no public specification. Dell EMC has never published the OneFS on-disk format. No open-source reverse-engineered reader exists. Without a spec we cannot honour the CONTRIBUTING rule "never advertise capabilities you cannot prove against a real spec". FreeBSD/UFS ancestry does NOT give us a UFS reader fallback. OneFS runs on a FreeBSD-derived kernel, but the filesystem layer is entirely proprietary — it is NOT FFS/UFS at the on-disk level. UFS1 places its superblock magic `0x00011954` at offset 8192; OneFS images have the ASCII `"OneFS"` tag at offset 0 and no UFS superblock. Routing OneFS images through `UfsReader` would fail the magic check and (if forced) return arbitrary bytes — the textbook mutual-compensation trap. Conclusion: Stage-0 detection only. Surface the magic, raw bytes, and a `metadata.ini` documenting the limitation. R/O promotion is blocked on (a) Dell EMC publishing the spec and (b) a multi-node ingest path — neither is in reach. References: Dell EMC "PowerScale OneFS Technical Overview" whitepaper — high-level architecture only; no on-disk spec is published`https://en.wikipedia.org/wiki/OneFS_distributed_file_system` — Wikipedia article +Conservative inspection descriptor for Dell PowerScale / Isilon OneFS media. -Implements `IArchiveFormatOperations`, `IFormatDescriptor`. +Implements `IArchiveFormatOperations`, `IFormatDescriptor`, `ILayoutOptimizable`. | Member | Signature | Summary | | --- | --- | --- | | `OneFsFormatDescriptor` | `OneFsFormatDescriptor()` | | -| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. | -| `Category` | `FormatCategory Category { get; }` | Gets the category. | -| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. | -| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. | -| `Description` | `string Description { get; }` | Gets the description. | -| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | -| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. | -| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. | -| `Id` | `string Id { get; }` | Gets the id. | -| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. | -| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. | -| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | -| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | -| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the conservative inspection capabilities. Integrity testing is not advertised: without a verified superblock/tree parser, successfully copying an opaque image does not prove that the filesystem is structurally sound. | +| `Category` | `FormatCategory Category { get; }` | Gets the format category. | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets compound extensions. | +| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the conventional extension for explicitly supplied raw media. | +| `Description` | `string Description { get; }` | Gets a description of the intentionally limited inspection surface. | +| `DisplayName` | `string DisplayName { get; }` | Gets the display name. | +| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the recognized extensions. | +| `Family` | `AlgorithmFamily Family { get; }` | Gets the algorithm family. | +| `Id` | `string Id { get; }` | Gets the registry id. | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets fixed magic signatures. Dell does not publish an authoritative offset-zero raw-media signature, so OneFS intentionally has none. | +| `Methods` | `IReadOnlyList Methods { get; }` | Gets the pseudo-archive storage method. | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. | +| `AnalyzeLayout` | `LayoutAnalysis AnalyzeLayout(Stream image)` | Reports the fixed physical geometry documented by Dell without reading or interpreting proprietary allocation metadata. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Extracts selected inspection entries through bounded streams. | +| `List` | `List List(Stream stream, string password)` | Lists the two conservative inspection entries without reading the image payload. | #### `OneFsReader` -Stage 0 detection-only reader for Dell EMC Isilon OneFS LIN-tree root images. OneFS is a clustered scale-out NAS — its single-image surface is the LIN-tree root block, whose first bytes are the ASCII tag `"OneFS"` (5 bytes, 0x4F 0x6E 0x65 0x46 0x53) or the short `"ONEF"` tag (0x4F 0x4E 0x45 0x46 = 0x4F4E4546 BE int) used in some node-local boot images. Only the tag is verified; the real LIN tree (logical inode number tree) is a cluster-wide construct and cannot be walked from a single image. File data is FEC-striped across nodes (N+M:B protection groups) — even a complete single-drive image carries only one stripe and cannot reconstruct file content without peer nodes. OneFS shares OS ancestry with FreeBSD, but the on-disk filesystem layer is proprietary and NOT UFS-compatible: there is no UFS1 superblock magic (`0x00011954`) at the UFS1 superblock offset (8192). The OneFS on-disk format has never been publicly specified by Dell EMC. +Conservative single-image inspection surface for Dell PowerScale / Isilon OneFS media. Implements `IDisposable`. | Member | Signature | Summary | | --- | --- | --- | -| `OneFsReader` | `OneFsReader(Stream stream)` | Initializes a new instance of `OneFsReader`. | -| `LongTag` | `static readonly byte[] LongTag` | OneFS LIN-tree-root long tag: ASCII "OneFS" (5 bytes). | -| `ShortTag` | `static readonly byte[] ShortTag` | OneFS short tag: ASCII "ONEF" (4 bytes, 0x4F4E4546 BE). | -| `Entries` | `IReadOnlyList Entries { get; }` | Gets the entries. | -| `Tag` | `string Tag { get; }` | Gets or sets the tag. | -| `TrailingWord` | `uint TrailingWord { get; }` | Gets or sets the trailing word. | -| `ValidHeader` | `bool ValidHeader { get; }` | Gets a value indicating whether valid header. | -| `Dispose` | `void Dispose()` | Releases resources held by this instance. | -| `Extract` | `byte[] Extract(OneFsEntry entry)` | Decodes the supplied input. | +| `OneFsReader` | `OneFsReader(Stream stream)` | Initializes a conservative OneFS image reader without consuming or owning the source stream. Kept as the original one-argument public constructor for binary/source API compatibility. | +| `OneFsReader` | `OneFsReader(Stream stream, bool leaveOpen)` | Initializes a conservative OneFS image reader without consuming the image. | +| `BlocksPerCylinderGroup` | `const int BlocksPerCylinderGroup` | Documented number of 8 KiB blocks in a 32 MiB cylinder group. | +| `CylinderGroupSize` | `const int CylinderGroupSize` | Documented physical cylinder-group size of each OneFS data disk. | +| `MetadataEntryName` | `const string MetadataEntryName` | Name of the synthetic inspection metadata entry. | +| `PhysicalBlockSize` | `const int PhysicalBlockSize` | Documented OneFS filesystem block size. | +| `RawImageEntryName` | `const string RawImageEntryName` | Name of the opaque raw-image entry. | +| `Entries` | `IReadOnlyList Entries { get; }` | Gets the synthetic entries exposed by the conservative reader. | +| `ImageSize` | `long ImageSize { get; }` | Gets the byte length of the opaque source image. | +| `Tag` | `string Tag { get; }` | Legacy compatibility property. No authoritative fixed OneFS raw-media tag is currently known, so this value is always empty. | +| `TrailingWord` | `uint TrailingWord { get; }` | Legacy compatibility property. No undocumented trailing header word is interpreted; this value is always zero. | +| `ValidHeader` | `bool ValidHeader { get; }` | Legacy compatibility property. The current reader does not claim to have validated a proprietary raw-media header, so this value is always false. | +| `Dispose` | `void Dispose()` | Releases the source stream when ownership was requested. | +| `Extract` | `byte[] Extract(OneFsEntry entry)` | Materializes an entry in memory. Prefer `OpenEntry` for the raw image so large media remains streaming. | +| `OpenEntry` | `Stream OpenEntry(OneFsEntry entry)` | Opens an entry as a bounded read-only stream. Opening the raw image resets the source to byte zero but never copies the payload into managed memory. | ### Namespace `FileSystem.OpenVms` @@ -11446,7 +11529,7 @@ WORM writer for QNX6 (Neutrino) filesystem images. Emits a power-safe layout: th ### Namespace `FileSystem.Refs` -[`RefsBlockMover`](#refsblockmover) · [`RefsExtentMap`](#refsextentmap) · [`RefsFormatDescriptor`](#refsformatdescriptor) +[`RefsBlockMover`](#refsblockmover) · [`RefsExtentMap`](#refsextentmap) · [`RefsFormatDescriptor`](#refsformatdescriptor) · [`RefsOfflineBlockCloner`](#refsofflineblockcloner) #### `RefsBlockMover` @@ -11507,6 +11590,14 @@ Implements `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveModifi | `ProbeFilesystem` | `FilesystemDriverProfile ProbeFilesystem(Stream image)` | Probes the image and reports the filesystem driver profile. | | `Remove` | `void Remove(Stream archive, string[] entryNames)` | Removes regular files or empty directories from an unmounted ReFS image. Namespace deletion is published through immutable B+ replacement pages and the alternate CHKP. | +#### `RefsOfflineBlockCloner` + +Performs the bounded offline ReFS block-clone operation for two existing, equal-sized ordinary extent-backed files. The destination's stream mapping is replaced by the source mapping through immutable object/Object-Table CoW, while root #6 gains the corresponding shared-cluster reference counts in the same alternate-checkpoint publication. This is deliberately narrower than FSCTL_DUPLICATE_EXTENTS_TO_FILE: it clones a whole cluster-aligned file only, does not create names or extend EOF, and refuses resident, sparse, integrity, already-shared and otherwise undecoded stream layouts. Native mounted-driver/MLog semantics remain fail-closed. + +| Member | Signature | Summary | +| --- | --- | --- | +| `CloneWholeFile` | `static void CloneWholeFile(Stream image, string sourcePath, string destinationPath)` | Replaces the contents of an existing destination file with a metadata-only clone of an existing source file on an unmounted ReFS image. | + ### Namespace `FileSystem.Reiser4` [`Reiser4BlockMover`](#reiser4blockmover) · [`Reiser4FormatDescriptor`](#reiser4formatdescriptor) · [`Reiser4MasterSb`](#reiser4mastersb) · [`Reiser4Reader`](#reiser4reader) · [`Reiser4Reader.Entry`](#reiser4readerentry) · [`Reiser4Writer`](#reiser4writer) @@ -13291,9 +13382,9 @@ Represents a tux 3 entry. #### `Tux3FormatDescriptor` -Read-only native-superblock descriptor for the linux-tux3 research filesystem. +Native-superblock descriptor for the linux-tux3 research filesystem. -Implements `IArchiveFormatOperations`, `IFormatDescriptor`, `ISyntheticEntryNames`. +Implements `IArchiveFormatOperations`, `IArchiveShrinkable`, `IFilesystemExtentMap`, `IFormatDescriptor`, `ISyntheticEntryNames`, `IWipeEmpty`. | Member | Signature | Summary | | --- | --- | --- | @@ -13311,8 +13402,10 @@ Implements `IArchiveFormatOperations`, `IFormatDescriptor`, `ISyntheticEntryName | `Methods` | `IReadOnlyList Methods { get; }` | | | `SyntheticEntryNames` | `IReadOnlySet SyntheticEntryNames { get; }` | | | `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `EnumerateExtents` | `IEnumerable EnumerateExtents(Stream image)` | Enumerates the provable byte layout without guessing at undecoded TUX3 allocation state. The declared native volume is reserved wholesale; only trailing bytes outside it are free. Invalid or arithmetically unrepresentable volume metadata reserves the complete physical image. | | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | | `List` | `List List(Stream stream, string password)` | | +| `Shrink` | `void Shrink(Stream input, Stream output)` | Removes only bytes beyond the volume size declared by `volblocks * blocksize`. Malformed, truncated, or arithmetically invalid images are copied through unchanged. | #### `Tux3Reader` @@ -14147,7 +14240,7 @@ Implements `IDisposable`. ### Namespace `FileSystem.Xfs` -[`XfsBlockMover`](#xfsblockmover) · [`XfsEntry`](#xfsentry) · [`XfsExtentMap`](#xfsextentmap) · [`XfsFilesystemDriverAdapter`](#xfsfilesystemdriveradapter) · [`XfsFormatDescriptor`](#xfsformatdescriptor) · [`XfsInPlaceAdder`](#xfsinplaceadder) · [`XfsModifier`](#xfsmodifier) · [`XfsReader`](#xfsreader) · [`XfsWriter`](#xfswriter) +[`XfsBlockMover`](#xfsblockmover) · [`XfsEntry`](#xfsentry) · [`XfsExtendedAttributes`](#xfsextendedattributes) · [`XfsExtentMap`](#xfsextentmap) · [`XfsFilesystemDriverAdapter`](#xfsfilesystemdriveradapter) · [`XfsFormatDescriptor`](#xfsformatdescriptor) · [`XfsInPlaceAdder`](#xfsinplaceadder) · [`XfsModifier`](#xfsmodifier) · [`XfsReader`](#xfsreader) · [`XfsWriter`](#xfswriter) #### `XfsBlockMover` @@ -14179,6 +14272,16 @@ Represents a xfs entry. | `Name` | `string Name { get; init; }` | Gets or sets the name. | | `Size` | `long Size { get; init; }` | Gets or sets the size. | +#### `XfsExtendedAttributes` + +Reads and mutates XFS short-form extended attributes stored in an inode's attribute fork. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Read` | `static IReadOnlyDictionary Read(Stream image, string path)` | Reads all short-form xattrs for `path`. | +| `Remove` | `static bool Remove(Stream image, string path, string name)` | Removes one short-form xattr, returning false when it was absent. | +| `Set` | `static void Set(Stream image, string path, string name, ReadOnlySpan value)` | Creates or replaces a short-form xattr. If the resulting fork does not fit in the inode, the call fails without modifying the image. | + #### `XfsExtentMap` Walks an XFS image and yields its actual on-disk byte layout. Targets the WORM writer profile: per-AG superblock + AGF + AGI + AGFL + bnobt/cntbt/ inobt headers as MetadataReserved, plus per-file extents (BMBT_REC packed 128-bit format) as Used runs. For inodes whose data fork is in `local` (inline) format, the file content lives inside the inode itself and surfaces as MetadataReserved. Streaming: never loads the whole image. All reads flow through a `SectorCache` so multi-TB XFS images (an XFS volume can span thousands of AGs) work without OOM.