diff --git a/Compression.Tests/Compression.Tests.csproj b/Compression.Tests/Compression.Tests.csproj index 023dbbe39..a6ad0d1fd 100644 --- a/Compression.Tests/Compression.Tests.csproj +++ b/Compression.Tests/Compression.Tests.csproj @@ -146,6 +146,10 @@ ZfsVectors.%(Filename)%(Extension) + + + SeqBoxVectors.%(Filename)%(Extension) + G722Vectors.%(Filename)%(Extension) diff --git a/Compression.Tests/ConversionMatrix/ConversionMatrixTests.cs b/Compression.Tests/ConversionMatrix/ConversionMatrixTests.cs index 664cfe99d..5fbc3a8ac 100644 --- a/Compression.Tests/ConversionMatrix/ConversionMatrixTests.cs +++ b/Compression.Tests/ConversionMatrix/ConversionMatrixTests.cs @@ -125,6 +125,7 @@ public static IEnumerable Grid() { { "StuffItX", "[single-payload/whole-image target] writer emits an image that re-lists as 0 files" }, { "Umx", "[single-payload/whole-image target] Unreal package; writer emits a blob that re-lists as 0 files" }, { "Wbn", "[single-payload/whole-image target] WebBundle writer collapses tree to a single FULL.wbn" }, + { "SeqBox", "[single-payload/whole-image target] a SeqBox container holds exactly one file; a tree of several is refused" }, { "Wrapster", "[single-payload/whole-image target] Wrapster MP3 wrapper; one FULL.mp3 + 0/1 frame, not a tree" }, // -- bucket: name/charset/size constraint -- { "Bbc", "[name/charset/size constraint] BBC Micro DFS disk; name-synthesizing FS, payload not found by content" }, diff --git a/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs new file mode 100644 index 000000000..bbbc898af --- /dev/null +++ b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs @@ -0,0 +1,37 @@ +using Compression.Registry; +using FileFormat.FirmwareHex; + +namespace Compression.Tests.FirmwareHex; + +[TestFixture] +public sealed class FirmwareHexPayloadTests { + [TestCase(false)] + [TestCase(true)] + public void Create_RejectsMultiplePayloadsWithoutWritingOutput(bool emptyFirst) { + var descriptor = new IntelHexFormatDescriptor(); + using var output = new MemoryStream(); + var inputs = new[] { + ArchiveInputInfo.InMemory("first.bin", emptyFirst ? [] : [1, 2, 3]), + ArchiveInputInfo.InMemory("second.bin", [9, 8, 7]), + }; + + Assert.Throws(() => + ((IArchiveCreatable)descriptor).Create(output, inputs, new FormatCreateOptions())); + Assert.That(output.Length, Is.Zero); + } + + [Test] + public void Add_RejectsAnotherPayloadWithoutChangingTheArchive() { + var descriptor = new IntelHexFormatDescriptor(); + using var archive = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(archive, + [ArchiveInputInfo.InMemory("firmware.bin", [1, 2, 3])], new FormatCreateOptions()); + var original = archive.ToArray(); + archive.Position = 0; + + Assert.Throws(() => ((IArchiveModifiable)descriptor).Add(archive, + [ArchiveInputInfo.InMemory("ADD.BIN", [9, 8, 7])])); + Assert.That(archive.ToArray(), Is.EqualTo(original)); + } + +} diff --git a/Compression.Tests/SeqBox/ReferenceVectors/README.md b/Compression.Tests/SeqBox/ReferenceVectors/README.md new file mode 100644 index 000000000..62894d80a --- /dev/null +++ b/Compression.Tests/SeqBox/ReferenceVectors/README.md @@ -0,0 +1,20 @@ +# SeqBox reference containers + +Every file here was written by the reference encoder `sbxenc.py` 1.0.2 from +[MarcoPon/SeqBox](https://github.com/MarcoPon/SeqBox) at revision +`dee472acfe89f08b7112ce854ca98f2094fbae3b`, run with Python 3.12 on Ubuntu 24.04 (WSL), never by +CompressionWorkbench. + +Sources: `fox.txt` = `The quick brown fox jumps over the lazy dog. ` × 50 (2250 bytes), `noise.bin` = +the 1000 bytes `(i * 31 + 7) mod 256`, `empty.bin` = 0 bytes; all with mtime 2001-02-03 04:05:06 UTC. + +| File | Command | What it pins | +| --- | --- | --- | +| `fox-v1.sbx` | `sbxenc.py -uid 0000c0ffee01 fox.txt fox-v1.sbx` | version 1 (512-byte blocks), full metadata block | +| `noise-v2.sbx` | `sbxenc.py -uid 0000c0ffee02 -sv 2 noise.bin noise-v2.sbx` | version 2 (128-byte blocks) | +| `fox-v3.sbx` | `sbxenc.py -uid 0000c0ffee03 -sv 3 fox.txt fox-v3.sbx` | version 3 (4096-byte blocks) | +| `noise-v4.sbx` | `sbxenc.py -uid 0000c0ffee04 -sv 4 noise.bin noise-v4.sbx` | version 4 (488-byte blocks) | +| `fox-nometa-v1.sbx` | `sbxenc.py -uid 0000c0ffee05 -nm fox.txt fox-nometa-v1.sbx` | no metadata block: payload keeps the 0x1A padding | +| `empty-v1.sbx` | `sbxenc.py -uid 0000c0ffee06 empty.bin empty-v1.sbx` | metadata block only | + +`sbxdec.py -i fox-v1.sbx` reports the expected name, size (2250) and SHA-256. diff --git a/Compression.Tests/SeqBox/ReferenceVectors/empty-v1.sbx b/Compression.Tests/SeqBox/ReferenceVectors/empty-v1.sbx new file mode 100644 index 000000000..252ee4d75 Binary files /dev/null and b/Compression.Tests/SeqBox/ReferenceVectors/empty-v1.sbx differ diff --git a/Compression.Tests/SeqBox/ReferenceVectors/fox-nometa-v1.sbx b/Compression.Tests/SeqBox/ReferenceVectors/fox-nometa-v1.sbx new file mode 100644 index 000000000..cf41fd472 Binary files /dev/null and b/Compression.Tests/SeqBox/ReferenceVectors/fox-nometa-v1.sbx differ diff --git a/Compression.Tests/SeqBox/ReferenceVectors/fox-v1.sbx b/Compression.Tests/SeqBox/ReferenceVectors/fox-v1.sbx new file mode 100644 index 000000000..0c8fd10de Binary files /dev/null and b/Compression.Tests/SeqBox/ReferenceVectors/fox-v1.sbx differ diff --git a/Compression.Tests/SeqBox/ReferenceVectors/fox-v3.sbx b/Compression.Tests/SeqBox/ReferenceVectors/fox-v3.sbx new file mode 100644 index 000000000..dd7a1a94e Binary files /dev/null and b/Compression.Tests/SeqBox/ReferenceVectors/fox-v3.sbx differ diff --git a/Compression.Tests/SeqBox/ReferenceVectors/noise-v2.sbx b/Compression.Tests/SeqBox/ReferenceVectors/noise-v2.sbx new file mode 100644 index 000000000..f8a6f2359 Binary files /dev/null and b/Compression.Tests/SeqBox/ReferenceVectors/noise-v2.sbx differ diff --git a/Compression.Tests/SeqBox/ReferenceVectors/noise-v4.sbx b/Compression.Tests/SeqBox/ReferenceVectors/noise-v4.sbx new file mode 100644 index 000000000..340b73063 Binary files /dev/null and b/Compression.Tests/SeqBox/ReferenceVectors/noise-v4.sbx differ diff --git a/Compression.Tests/SeqBox/SeqBoxTests.cs b/Compression.Tests/SeqBox/SeqBoxTests.cs new file mode 100644 index 000000000..8d6a620c7 --- /dev/null +++ b/Compression.Tests/SeqBox/SeqBoxTests.cs @@ -0,0 +1,203 @@ +using System.Reflection; +using Compression.Registry; +using FileFormat.SeqBox; + +namespace Compression.Tests.SeqBox; + +/// +/// SeqBox (.sbx): containers written by the reference sbxenc (SeqBox/ReferenceVectors), our +/// writer, and the reference sbxdec as a live oracle for what we write. +/// +[TestFixture] +public class SeqBoxTests { + + private static readonly byte[] FoxText = + System.Text.Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat("The quick brown fox jumps over the lazy dog. ", 50))); + + private static readonly byte[] Noise = Enumerable.Range(0, 1000).Select(i => (byte)((i * 31 + 7) % 256)).ToArray(); + + private static readonly long FoxTime = new DateTimeOffset(2001, 2, 3, 4, 5, 6, TimeSpan.Zero).ToUnixTimeSeconds(); + + private string _tmpDir = null!; + + [SetUp] + public void Setup() { + this._tmpDir = Path.Combine(Path.GetTempPath(), $"cwb_sbx_{Guid.NewGuid():N}"); + Directory.CreateDirectory(this._tmpDir); + } + + [TearDown] + public void Teardown() { + try { Directory.Delete(this._tmpDir, true); } catch { /* best effort */ } + } + + // ── reference containers (sbxenc 1.0.2) ───────────────────────────── + + [TestCase("fox-v1.sbx", 1, "fox.txt")] + [TestCase("fox-v3.sbx", 3, "fox.txt")] + [TestCase("noise-v2.sbx", 2, "noise.bin")] + [TestCase("noise-v4.sbx", 4, "noise.bin")] + [Category("ThemVsUs")] + public void GivenSbxencContainer_WhenRead_ThenMetadataAndPayloadMatchTheSource(string vector, int version, string name) { + var p = SeqBoxContainer.Read(new MemoryStream(LoadVector(vector))); + + Assert.That(p.Version, Is.EqualTo(version)); + Assert.That(p.Metadata, Is.Not.Null); + Assert.That(p.Metadata!.FileName, Is.EqualTo(name)); + Assert.That(p.Metadata.SbxName, Is.EqualTo(vector)); + Assert.That(p.Metadata.FileTime, Is.EqualTo(FoxTime)); + Assert.That(p.Data, Is.EqualTo(name == "fox.txt" ? FoxText : Noise)); + } + + [Test, Category("ThemVsUs")] + public void GivenSbxencContainerWithoutMetadata_WhenRead_ThenPayloadKeepsTheLastBlocksPadding() { + var p = SeqBoxContainer.Read(new MemoryStream(LoadVector("fox-nometa-v1.sbx"))); + + Assert.That(p.Metadata, Is.Null); + Assert.That(p.DataBlocks, Is.EqualTo(5)); + Assert.That(p.Data.AsSpan(0, FoxText.Length).ToArray(), Is.EqualTo(FoxText)); + Assert.That(p.Data.Skip(FoxText.Length), Has.All.EqualTo((byte)0x1A)); + } + + [Test, Category("Boundary")] + public void GivenSbxencContainerOfAnEmptyFile_WhenRead_ThenOnlyTheMetadataBlockAndAnEmptyPayload() { + var p = SeqBoxContainer.Read(new MemoryStream(LoadVector("empty-v1.sbx"))); + + Assert.That(p.DataBlocks, Is.Zero); + Assert.That(p.Metadata!.FileSize, Is.Zero); + Assert.That(p.Data, Is.Empty); + } + + [Test, Category("ThemVsUs")] + public void GivenSbxencContainer_WhenListedAndExtractedThroughTheDescriptor_ThenOneNamedEntry() { + var d = new SeqBoxFormatDescriptor(); + var bytes = LoadVector("fox-v1.sbx"); + + var list = d.List(new MemoryStream(bytes), null); + Assert.That(list.Single().Name, Is.EqualTo("fox.txt")); + Assert.That(list.Single().OriginalSize, Is.EqualTo(FoxText.Length)); + + d.Extract(new MemoryStream(bytes), this._tmpDir, null, null); + Assert.That(File.ReadAllBytes(Path.Combine(this._tmpDir, "fox.txt")), Is.EqualTo(FoxText)); + } + + // ── our writer ────────────────────────────────────────────────────── + + [TestCase(1, 512)] + [TestCase(2, 128)] + [TestCase(3, 4096)] + [TestCase(4, 488)] + [Category("RoundTrip")] + public void GivenData_WhenWrittenAtEachVersion_ThenBlockAlignedAndRoundTrips(int version, int blockSize) { + var ms = new MemoryStream(); + SeqBoxContainer.Write(ms, Noise, "noise.bin", version, uid: [0, 0, 0xC0, 0xFF, 0xEE, 0x09]); + + Assert.That(ms.Length % blockSize, Is.Zero); + var p = SeqBoxContainer.Read(new MemoryStream(ms.ToArray())); + Assert.That(p.Version, Is.EqualTo(version)); + Assert.That(p.Uid, Is.EqualTo(new byte[] { 0, 0, 0xC0, 0xFF, 0xEE, 0x09 })); + Assert.That(p.Data, Is.EqualTo(Noise)); + } + + [Test, Category("Boundary")] + public void GivenPayloadOfExactlyOneBlock_WhenWritten_ThenNoPaddingBlockFollows() { + var data = Enumerable.Range(0, 496).Select(i => (byte)i).ToArray(); + var ms = new MemoryStream(); + SeqBoxContainer.Write(ms, data, "x.bin"); + + Assert.That(ms.Length, Is.EqualTo(2 * 512)); + Assert.That(SeqBoxContainer.Read(new MemoryStream(ms.ToArray())).Data, Is.EqualTo(data)); + } + + [Test, Category("Boundary")] + public void GivenEmptyFile_WhenWritten_ThenOnlyTheMetadataBlock() { + var ms = new MemoryStream(); + SeqBoxContainer.Write(ms, [], "empty.bin"); + + Assert.That(ms.Length, Is.EqualTo(512)); + Assert.That(SeqBoxContainer.Read(new MemoryStream(ms.ToArray())).Data, Is.Empty); + } + + [Test, Category("Exception")] + public void GivenTwoFiles_WhenCreated_ThenRefusesBecauseSeqBoxHoldsOneFile() { + var d = new SeqBoxFormatDescriptor(); + Assert.That(() => d.Create(new MemoryStream(), + [ArchiveInputInfo.InMemory("a", [1]), ArchiveInputInfo.InMemory("b", [2])], new FormatCreateOptions()), + Throws.InstanceOf()); + } + + [Test, Category("Exception")] + public void GivenFlippedDataByte_WhenRead_ThenTheBlockCrcRejectsIt() { + var bytes = LoadVector("fox-v1.sbx"); + bytes[512 * 2 + 100] ^= 0x01; + + Assert.That(() => SeqBoxContainer.Read(new MemoryStream(bytes)), Throws.InstanceOf().With.Message.Contains("CRC")); + } + + [Test, Category("Exception")] + public void GivenSwappedBlocks_WhenRead_ThenTheSequenceCheckRejectsIt() { + var bytes = LoadVector("fox-v1.sbx"); + var swapped = bytes.ToArray(); + bytes.AsSpan(512, 512).CopyTo(swapped.AsSpan(1024)); + bytes.AsSpan(1024, 512).CopyTo(swapped.AsSpan(512)); + + Assert.That(() => SeqBoxContainer.Read(new MemoryStream(swapped)), Throws.InstanceOf().With.Message.Contains("order")); + } + + [Test, Category("Exception")] + public void GivenMissingLastBlock_WhenRead_ThenTheFileSizeCheckRejectsIt() { + var bytes = LoadVector("fox-v1.sbx"); + + Assert.That(() => SeqBoxContainer.Read(new MemoryStream(bytes[..^512])), Throws.InstanceOf()); + } + + [Test, Category("Exception")] + public void GivenUnknownVersion_WhenRead_ThenRefused() { + var bytes = LoadVector("fox-v1.sbx"); + bytes[3] = 9; + + Assert.That(() => SeqBoxContainer.Read(new MemoryStream(bytes)), Throws.InstanceOf()); + Assert.That(SeqBoxContainer.IsSeqBox(bytes), Is.False); + } + + // ── oracle: the reference decoder reads what we write ─────────────── + + [TestCase(1)] + [TestCase(2)] + [TestCase(3)] + [TestCase(4)] + [Category("ExternalInterop")] + public void GivenOurContainer_WhenSbxdecDecodesIt_ThenHashMatchesAndBytesAreIdentical(int version) { + var sbxdec = RequireSbxdec(); + var big = Enumerable.Range(0, 20_000).Select(i => (byte)(i * 7 + (i >> 8))).ToArray(); + var path = Path.Combine(this._tmpDir, "ours.sbx"); + using (var f = File.Create(path)) + SeqBoxContainer.Write(f, big, "big.bin", version); + var outPath = Path.Combine(this._tmpDir, "decoded.bin"); + + var r = FsInteropToolbox.RunWsl( + $"python3 {sbxdec} -o '{FsInteropToolbox.WinToWsl(path)}' '{FsInteropToolbox.WinToWsl(outPath)}'"); + + Assert.That(r.ExitCode, Is.EqualTo(0), $"sbxdec rejected our container:\n{r.StdOut}\n{r.StdErr}"); + Assert.That(r.StdOut, Does.Contain("hash match!")); + Assert.That(File.ReadAllBytes(outPath), Is.EqualTo(big)); + } + + private static string RequireSbxdec() { + if (OperatingSystem.IsWindows() && !FsInteropToolbox.WslAvailable) + Assert.Ignore("WSL is not available."); + var dir = Environment.GetEnvironmentVariable("SEQBOX_HOME") ?? "/opt/seqbox"; + var probe = FsInteropToolbox.RunWsl($"test -f '{dir}/sbxdec.py' && command -v python3"); + if (probe.ExitCode != 0) + Assert.Ignore($"SeqBox reference tools not found at {dir} (set SEQBOX_HOME to a checkout of github.com/MarcoPon/SeqBox)."); + return $"'{dir}/sbxdec.py'"; + } + + private static byte[] LoadVector(string name) { + using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream($"SeqBoxVectors.{name}") + ?? throw new InvalidOperationException($"missing embedded vector {name}"); + using var ms = new MemoryStream(); + stream.CopyTo(ms); + return ms.ToArray(); + } +} diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.SeqBox/SeqBoxContainer.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.SeqBox/SeqBoxContainer.cs new file mode 100644 index 000000000..0d14ac933 --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.SeqBox/SeqBoxContainer.cs @@ -0,0 +1,239 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; +using System.Text; +using Compression.Core.Checksums; + +namespace FileFormat.SeqBox; + +/// Metadata carried by a SeqBox block 0 (all fields optional). +/// FNM: the encoded file's name (UTF-8). +/// SNM: the container's own file name (UTF-8). +/// FSZ: the encoded file's size; trims the last block's 0x1A padding. +/// FDT: the encoded file's modification time, seconds since the Unix epoch. +/// SDT: when the container was made, seconds since the Unix epoch. +/// HSH: a multihash; 0x12 0x20 + 32 bytes is SHA-256. +public sealed record SeqBoxMetadata( + string? FileName = null, + string? SbxName = null, + long? FileSize = null, + long? FileTime = null, + long? SbxTime = null, + byte[]? Hash = null); + +/// +/// SeqBox (SBX) single-file container: the file is cut into fixed-size blocks, each with a +/// 16-byte header ("SBx", version, CRC-16, 6-byte file UID, 32-bit sequence number, all +/// big-endian) so the file can be recovered from a damaged or unindexed disk by scanning for +/// blocks. Block 0 optionally carries TLV metadata; data blocks start at sequence 1, and the last +/// one is padded with 0x1A. The CRC is CRC-16/CCITT (polynomial 0x1021, MSB first) seeded with +/// the version number and taken over everything after the CRC field. +/// +/// +/// Layout from the format author's description (github.com/MarcoPon/SeqBox, README); the +/// reference sbxenc/sbxdec were used only as a black-box oracle. +/// +public static class SeqBoxContainer { + /// Size of every block header. + public const int HeaderSize = 16; + + /// The three signature bytes every block starts with. + public static ReadOnlySpan Signature => "SBx"u8; + + /// Multihash prefix for SHA-256 (code 0x12, length 32). + private static ReadOnlySpan Sha256Multihash => [0x12, 0x20]; + + /// Block size of a SeqBox version, or 0 for an unknown version. + public static int BlockSize(int version) => version switch { + 1 => 512, + 2 => 128, + 3 => 4096, + 4 => 488, // Amiga OFS data-block payload + _ => 0, + }; + + /// Whether the stream starts with a valid SeqBox block (signature, known version, CRC). + public static bool IsSeqBox(ReadOnlySpan head) { + if (head.Length < HeaderSize || !head[..3].SequenceEqual(Signature)) + return false; + var size = BlockSize(head[3]); + return size > 0 && head.Length >= size && CrcOk(head[..size]); + } + + private static ushort Crc(ReadOnlySpan blockAfterCrc, int version) { + var crc = new Crc16Ccitt((ushort)version); + crc.Update(blockAfterCrc); + return (ushort)crc.Value; + } + + private static bool CrcOk(ReadOnlySpan block) + => BinaryPrimitives.ReadUInt16BigEndian(block[4..]) == Crc(block[6..], block[3]); + + /// A parsed container: version, UID, metadata (if block 0 is present) and the payload. + public sealed record Parsed(int Version, byte[] Uid, SeqBoxMetadata? Metadata, int DataBlocks, long PayloadLength, byte[] Data); + + /// + /// Parses and verifies a whole container. Every block must carry the signature, the first + /// block's version and UID and a valid CRC; data blocks must be numbered 1, 2, 3, … in file + /// order. With FSZ the payload is trimmed to it; with an HSH SHA-256 the payload is verified. + /// Without FSZ the trailing 0x1A padding cannot be told from data and is kept, as sbxdec does. + /// + /// Any block is damaged, missing or out of order, or the hash does not match. + public static Parsed Read(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + if (stream.CanSeek) stream.Position = 0; + Span head = stackalloc byte[4]; + if (stream.ReadAtLeast(head, 4, throwOnEndOfStream: false) < 4 || !head[..3].SequenceEqual(Signature)) + throw new InvalidDataException("Not a SeqBox container (missing 'SBx' signature)."); + int version = head[3], blockSize = BlockSize(version); + if (blockSize == 0) + throw new InvalidDataException($"SeqBox block version {version} is not supported."); + + var block = new byte[blockSize]; + head.CopyTo(block); + var filled = 4; + byte[]? uid = null; + SeqBoxMetadata? metadata = null; + using var payload = new MemoryStream(); + long expectedSeq = 1; + var blockIndex = 0; + var dataBlocks = 0; + while (true) { + var got = stream.ReadAtLeast(block.AsSpan(filled), blockSize - filled, throwOnEndOfStream: false); + if (got + filled == 0) break; + if (got + filled < blockSize) { + if (blockIndex == 0) + throw new InvalidDataException("SeqBox container is shorter than one block."); + break; // sbxdec ignores a trailing partial block + } + filled = 0; + + if (!block.AsSpan(0, 3).SequenceEqual(Signature) || block[3] != version) + throw new InvalidDataException($"SeqBox block {blockIndex} has no valid signature."); + if (!CrcOk(block)) + throw new InvalidDataException($"SeqBox block {blockIndex} fails its CRC."); + var blockUid = block.AsSpan(6, 6).ToArray(); + if (uid is null) uid = blockUid; + else if (!blockUid.AsSpan().SequenceEqual(uid)) + throw new InvalidDataException($"SeqBox block {blockIndex} belongs to another file (UID mismatch)."); + + var seq = BinaryPrimitives.ReadUInt32BigEndian(block.AsSpan(12)); + if (seq == 0) { + if (blockIndex != 0) + throw new InvalidDataException($"SeqBox metadata block found at position {blockIndex}."); + metadata = ParseMetadata(block.AsSpan(HeaderSize)); + } else { + if (seq != expectedSeq) + throw new InvalidDataException($"SeqBox block {expectedSeq} is missing or out of order (found {seq})."); + ++expectedSeq; + ++dataBlocks; + payload.Write(block, HeaderSize, blockSize - HeaderSize); + } + ++blockIndex; + } + + var data = payload.ToArray(); + if (metadata?.FileSize is { } size) { + // The size has to land in the last data block: anything else means blocks are missing + // from the end or the metadata belongs to a different file. + var dataSize = blockSize - HeaderSize; + var fits = dataBlocks == 0 ? size == 0 : size > (long)(dataBlocks - 1) * dataSize && size <= data.Length; + if (!fits) + throw new InvalidDataException($"SeqBox file size {size} does not fit the {dataBlocks} data blocks present."); + data = data[..(int)size]; + } + if (metadata?.Hash is { Length: 34 } hash && hash.AsSpan(0, 2).SequenceEqual(Sha256Multihash) + && !SHA256.HashData(data).AsSpan().SequenceEqual(hash.AsSpan(2))) + throw new InvalidDataException("SeqBox SHA-256 does not match the decoded data."); + + return new(version, uid!, metadata, dataBlocks, data.Length, data); + } + + private static SeqBoxMetadata ParseMetadata(ReadOnlySpan area) { + string? fnm = null, snm = null; + long? fsz = null, fdt = null, sdt = null; + byte[]? hsh = null; + var p = 0; + while (p + 4 <= area.Length) { + var id = area.Slice(p, 3); + if (id[0] == 0x1A && id[1] == 0x1A && id[2] == 0x1A) break; + int len = area[p + 3]; + if (p + 4 + len > area.Length) + throw new InvalidDataException("SeqBox metadata entry runs past the block."); + var value = area.Slice(p + 4, len); + switch (Encoding.ASCII.GetString(id)) { + case "FNM": fnm = Encoding.UTF8.GetString(value); break; + case "SNM": snm = Encoding.UTF8.GetString(value); break; + case "FSZ": fsz = ReadBigEndian(value); break; + case "FDT": fdt = ReadBigEndian(value); break; + case "SDT": sdt = ReadBigEndian(value); break; + case "HSH": hsh = value.ToArray(); break; + // PID and unknown ids are skipped: the TLV length makes that safe. + } + p += 4 + len; + } + return new(fnm, snm, fsz, fdt, sdt, hsh); + } + + private static long ReadBigEndian(ReadOnlySpan value) { + if (value.Length is 0 or > 8) + throw new InvalidDataException("SeqBox numeric metadata must be 1-8 bytes."); + long v = 0; + foreach (var b in value) v = (v << 8) | b; + return v; + } + + /// + /// Writes a container the way sbxenc does: block 0 with FNM, SNM, FSZ, FDT, SDT and a SHA-256 + /// HSH (unless is false), then the data in sequence-numbered + /// blocks, the last padded with 0x1A. An empty file is the metadata block alone. + /// + public static void Write(Stream output, ReadOnlySpan data, string fileName, int version = 1, + byte[]? uid = null, DateTimeOffset? fileTime = null, string? sbxName = null, + DateTimeOffset? sbxTime = null, bool withMetadata = true) { + ArgumentNullException.ThrowIfNull(output); + var blockSize = BlockSize(version); + if (blockSize == 0) + throw new ArgumentOutOfRangeException(nameof(version), version, "SeqBox versions are 1 (512), 2 (128), 3 (4096) and 4 (488)."); + if (uid is { Length: not 6 }) + throw new ArgumentException("A SeqBox UID is 6 bytes.", nameof(uid)); + uid ??= RandomNumberGenerator.GetBytes(6); + var dataSize = blockSize - HeaderSize; + var block = new byte[blockSize]; + + if (withMetadata) { + var meta = new MemoryStream(); + void Tlv(string id, ReadOnlySpan value) { + if (value.Length > 255) + throw new NotSupportedException($"SeqBox metadata '{id}' is longer than 255 bytes."); + meta.Write(Encoding.ASCII.GetBytes(id)); + meta.WriteByte((byte)value.Length); + meta.Write(value); + } + static byte[] Be(long v) { var b = new byte[8]; BinaryPrimitives.WriteInt64BigEndian(b, v); return b; } + Tlv("FNM", Encoding.UTF8.GetBytes(fileName)); + if (sbxName is not null) Tlv("SNM", Encoding.UTF8.GetBytes(sbxName)); + Tlv("FSZ", Be(data.Length)); + Tlv("FDT", Be((fileTime ?? DateTimeOffset.UnixEpoch).ToUnixTimeSeconds())); + Tlv("SDT", Be((sbxTime ?? DateTimeOffset.UtcNow).ToUnixTimeSeconds())); + Tlv("HSH", [.. Sha256Multihash, .. SHA256.HashData(data)]); + if (meta.Length > dataSize) + throw new NotSupportedException($"SeqBox metadata ({meta.Length} bytes) does not fit a version-{version} block."); + EmitBlock(output, block, version, uid, 0, meta.ToArray()); + } + + uint seq = 0; + for (var offset = 0; offset < data.Length; offset += dataSize) + EmitBlock(output, block, version, uid, ++seq, data.Slice(offset, Math.Min(dataSize, data.Length - offset))); + } + + private static void EmitBlock(Stream output, byte[] block, int version, byte[] uid, uint seq, ReadOnlySpan content) { + block.AsSpan().Fill(0x1A); + Signature.CopyTo(block); + block[3] = (byte)version; + uid.CopyTo(block, 6); + BinaryPrimitives.WriteUInt32BigEndian(block.AsSpan(12), seq); + content.CopyTo(block.AsSpan(HeaderSize)); + BinaryPrimitives.WriteUInt16BigEndian(block.AsSpan(4), Crc(block.AsSpan(6), version)); + output.Write(block); + } +} diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.SeqBox/SeqBoxFormatDescriptor.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.SeqBox/SeqBoxFormatDescriptor.cs new file mode 100644 index 000000000..c9b749434 --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.SeqBox/SeqBoxFormatDescriptor.cs @@ -0,0 +1,91 @@ +using Compression.Registry; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.SeqBox; + +/// +/// SeqBox (.sbx), Marco Pontello's recoverable single-file container. Lists as one entry named by +/// the FNM metadata; creates the sbxenc layout (version 1, 512-byte blocks, full metadata). +/// +public sealed class SeqBoxFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable { + /// Entry name used when the container has no metadata block (sbxdec appends ".out"). + public const string UnnamedEntry = "sbx.out"; + + /// + public string Id => "SeqBox"; + /// + public string DisplayName => "SeqBox"; + /// + public FormatCategory Category => FormatCategory.Archive; + /// + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | FormatCapabilities.CanTest; + /// + public string DefaultExtension => ".sbx"; + /// + public IReadOnlyList Extensions => [".sbx"]; + /// + public IReadOnlyList CompoundExtensions => []; + /// + public IReadOnlyList MagicSignatures => [ + new("SBx\x01"u8.ToArray(), Confidence: 0.85), + new("SBx\x02"u8.ToArray(), Confidence: 0.85), + new("SBx\x03"u8.ToArray(), Confidence: 0.85), + new("SBx\x04"u8.ToArray(), Confidence: 0.85), + ]; + /// + public IReadOnlyList Methods => [new("sbx1", "512-byte blocks"), new("sbx2", "128-byte blocks"), new("sbx3", "4096-byte blocks"), new("sbx4", "488-byte blocks")]; + /// + public string? TarCompressionFormatId => null; + /// + public AlgorithmFamily Family => AlgorithmFamily.Archive; + /// + public string Description => "SeqBox recoverable container: file split into self-identifying, CRC-checked, sequence-numbered blocks"; + + private static string EntryName(SeqBoxContainer.Parsed p) + => string.IsNullOrEmpty(p.Metadata?.FileName) ? UnnamedEntry : Path.GetFileName(p.Metadata!.FileName.Replace('\\', '/')); + + /// + public List List(Stream stream, string? password) { + var p = SeqBoxContainer.Read(stream); + DateTime? modified = p.Metadata?.FileTime is { } t ? DateTimeOffset.FromUnixTimeSeconds(t).UtcDateTime : null; + return [new ArchiveEntryInfo(0, EntryName(p), p.PayloadLength, stream.Length, $"sbx{p.Version}", false, false, modified)]; + } + + /// + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + var p = SeqBoxContainer.Read(stream); + var name = EntryName(p); + if (files is { Length: > 0 } && !MatchesFilter(name, files)) + return; + WriteFile(outputDir, name, p.Data); + } + + /// + public byte[] ExtractEntryToMemory(Stream archive, string entryName, string? password) { + var p = SeqBoxContainer.Read(archive); + return string.Equals(EntryName(p), entryName, StringComparison.OrdinalIgnoreCase) ? p.Data : []; + } + + /// + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + var files = inputs.Where(i => !i.IsDirectory).ToList(); + if (files.Count != 1) + throw new NotSupportedException($"A SeqBox container holds exactly one file ({files.Count} given)."); + if (!string.IsNullOrEmpty(options.Password)) + throw new NotSupportedException("SeqBox password scrambling is not an encryption method and is not written."); + var version = (options.MethodName ?? "sbx1").Trim().ToLowerInvariant() switch { + "" or "sbx1" or "1" => 1, + "sbx2" or "2" => 2, + "sbx3" or "3" => 3, + "sbx4" or "4" => 4, + var m => throw new NotSupportedException($"SeqBox method '{m}' is not supported (sbx1-sbx4)."), + }; + var input = files[0]; + var name = Path.GetFileName(input.ArchiveName.Replace('\\', '/')); + var mtime = input.Metadata?.LastWriteTimeUtc; + SeqBoxContainer.Write(output, input.ReadContent(), name, version, fileTime: mtime); + } +} diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md index 9357f828e..3f20181fd 100644 --- a/Hawkynt.FileFormats.Archives/README.md +++ b/Hawkynt.FileFormats.Archives/README.md @@ -134,6 +134,7 @@ Every State, Test, Maintenance, Compress/Decompress and Demux/Mux/Remux cell is | [NuFX / ShrinkIt](https://en.wikipedia.org/wiki/ShrinkIt) | `NuFx` | `.shk` `.sdk` `.bxy` | R/W | ✅ | defrag · shrink · wipe | | [nulib.com](https://nulib.com/library/FTN.e08002.htm) | | PackIt | `PackIt` | `.pit` | R/W | ✅ | defrag · wipe | Reads stored and Huffman entries, writes stored; encrypted entries are refused | [GitHub](https://github.com/MacPaw/XADMaster) | | [RAR](https://en.wikipedia.org/wiki/RAR_(file_format)) | `Rar` | `.rar` | R/W | ✅ | wipe | v1–v5 readers; creation and edits emit RAR4/RAR5 without claiming WinRAR encoder parity | [rarlab.com](https://www.rarlab.com/technote.htm) | +| [SeqBox](https://github.com/MarcoPon/SeqBox) | `SeqBox` | `.sbx` | WORM | ✅ | — | Single-file recoverable container; versions 1-4, metadata block optional | [GitHub](https://github.com/MarcoPon/SeqBox) | | [7z](https://en.wikipedia.org/wiki/7z) | `SevenZip` | `.7z` | R/W | ✅ | compress · wipe | | [7-zip.org](https://www.7-zip.org/7z.html) | | [SHAR](https://en.wikipedia.org/wiki/Shar) | `Shar` | `.shar` `.sh` | R/W | ✅ | defrag | Add appends in place; Remove re-emits the script from the survivors | [gnu.org](https://www.gnu.org/software/sharutils/) | | [Spark (RISC OS)](https://en.wikipedia.org/wiki/ARC_(file_format)) | `Spark` | `.spk` `.spark` | R/W | ✅ | defrag · wipe | | [Archive Team](http://fileformats.archiveteam.org/wiki/Spark) | @@ -544,7 +545,7 @@ Use it when a .NET process needs to enumerate, extract, test, create, edit or in -Every public and protected member of all 2571 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). +Every public and protected member of all 2575 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md). diff --git a/Hawkynt.FileFormats.Archives/REFERENCE.md b/Hawkynt.FileFormats.Archives/REFERENCE.md index 464eb0ba6..6677ad5a4 100644 --- a/Hawkynt.FileFormats.Archives/REFERENCE.md +++ b/Hawkynt.FileFormats.Archives/REFERENCE.md @@ -26967,6 +26967,82 @@ Implements `IArchiveFormatOperations`, `IArchiveInMemoryExtract`, `IFormatDescri | `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. | | `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. | +### Namespace `FileFormat.SeqBox` + +[`SeqBoxContainer`](#seqboxcontainer) · [`SeqBoxContainer.Parsed`](#seqboxcontainerparsed) · [`SeqBoxFormatDescriptor`](#seqboxformatdescriptor) · [`SeqBoxMetadata`](#seqboxmetadata) + +#### `SeqBoxContainer` + +SeqBox (SBX) single-file container: the file is cut into fixed-size blocks, each with a 16-byte header (`"SBx"`, version, CRC-16, 6-byte file UID, 32-bit sequence number, all big-endian) so the file can be recovered from a damaged or unindexed disk by scanning for blocks. Block 0 optionally carries TLV metadata; data blocks start at sequence 1, and the last one is padded with 0x1A. The CRC is CRC-16/CCITT (polynomial 0x1021, MSB first) seeded with the version number and taken over everything after the CRC field. + +| Member | Signature | Summary | +| --- | --- | --- | +| `HeaderSize` | `const int HeaderSize` | Size of every block header. | +| `Signature` | `static ReadOnlySpan Signature { get; }` | The three signature bytes every block starts with. | +| `BlockSize` | `static int BlockSize(int version)` | Block size of a SeqBox version, or 0 for an unknown version. | +| `IsSeqBox` | `static bool IsSeqBox(ReadOnlySpan head)` | Whether the stream starts with a valid SeqBox block (signature, known version, CRC). | +| `Read` | `static Parsed Read(Stream stream)` | Parses and verifies a whole container. Every block must carry the signature, the first block's version and UID and a valid CRC; data blocks must be numbered 1, 2, 3, … in file order. With FSZ the payload is trimmed to it; with an HSH SHA-256 the payload is verified. Without FSZ the trailing 0x1A padding cannot be told from data and is kept, as sbxdec does. | +| `Write` | `static void Write(Stream output, ReadOnlySpan data, string fileName, int version = 1, byte[] uid = null, DateTimeOffset? fileTime = null, string sbxName = null, DateTimeOffset? sbxTime = null, bool withMetadata = true)` | Writes a container the way sbxenc does: block 0 with FNM, SNM, FSZ, FDT, SDT and a SHA-256 HSH (unless `withMetadata` is false), then the data in sequence-numbered blocks, the last padded with 0x1A. An empty file is the metadata block alone. | + +#### `SeqBoxContainer.Parsed` + +A parsed container: version, UID, metadata (if block 0 is present) and the payload. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Parsed` | `Parsed(int Version, byte[] Uid, SeqBoxMetadata Metadata, int DataBlocks, long PayloadLength, byte[] Data)` | A parsed container: version, UID, metadata (if block 0 is present) and the payload. | +| `DataBlocks` | `int DataBlocks { get; init; }` | | +| `Data` | `byte[] Data { get; init; }` | | +| `Metadata` | `SeqBoxMetadata Metadata { get; init; }` | | +| `PayloadLength` | `long PayloadLength { get; init; }` | | +| `Uid` | `byte[] Uid { get; init; }` | | +| `Version` | `int Version { get; init; }` | | + +#### `SeqBoxFormatDescriptor` + +SeqBox (.sbx), Marco Pontello's recoverable single-file container. Lists as one entry named by the FNM metadata; creates the sbxenc layout (version 1, 512-byte blocks, full metadata). + +Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IFormatDescriptor`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `SeqBoxFormatDescriptor` | `SeqBoxFormatDescriptor()` | | +| `UnnamedEntry` | `const string UnnamedEntry` | Entry name used when the container has no metadata block (sbxdec appends ".out"). | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | | +| `ExtractEntryToMemory` | `byte[] ExtractEntryToMemory(Stream archive, string entryName, string password)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | + +#### `SeqBoxMetadata` + +Metadata carried by a SeqBox block 0 (all fields optional). + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `SeqBoxMetadata` | `SeqBoxMetadata(string FileName = null, string SbxName = null, long? FileSize = null, long? FileTime = null, long? SbxTime = null, byte[] Hash = null)` | Metadata carried by a SeqBox block 0 (all fields optional). | +| `FileName` | `string FileName { get; init; }` | FNM: the encoded file's name (UTF-8). | +| `FileSize` | `long? FileSize { get; init; }` | FSZ: the encoded file's size; trims the last block's 0x1A padding. | +| `FileTime` | `long? FileTime { get; init; }` | FDT: the encoded file's modification time, seconds since the Unix epoch. | +| `Hash` | `byte[] Hash { get; init; }` | HSH: a multihash; `0x12 0x20` + 32 bytes is SHA-256. | +| `SbxName` | `string SbxName { get; init; }` | SNM: the container's own file name (UTF-8). | +| `SbxTime` | `long? SbxTime { get; init; }` | SDT: when the container was made, seconds since the Unix epoch. | + ### Namespace `FileFormat.SevenZip` [`SevenZipCodec`](#sevenzipcodec) · [`SevenZipEntry`](#sevenzipentry) · [`SevenZipFilter`](#sevenzipfilter) · [`SevenZipFormatDescriptor`](#sevenzipformatdescriptor) · [`SevenZipInPlaceAdder`](#sevenzipinplaceadder) · [`SevenZipInPlaceRemover`](#sevenzipinplaceremover) · [`SevenZipLayoutMap`](#sevenziplayoutmap) · [`SevenZipReader`](#sevenzipreader) · [`SevenZipWriter`](#sevenzipwriter) · [`SevenZipWriter.BlockDescriptor`](#sevenzipwriterblockdescriptor) · [`SolidBlockOptimizer`](#solidblockoptimizer) · [`SolidBlockOptimizer.DetailedProgress`](#solidblockoptimizerdetailedprogress) · [`SolidBlockOptimizer.OptimizeResult`](#solidblockoptimizeroptimizeresult) · [`SolidBlockOptimizer.ProgressCallback`](#solidblockoptimizerprogresscallback) · [`SolidBlockOptimizer.TrialResult`](#solidblockoptimizertrialresult) · [`SolidBlockPlanner`](#solidblockplanner) · [`SolidBlockPlanner.SolidBlock`](#solidblockplannersolidblock) diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs index 6c7d4b452..1c9e17266 100644 --- a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs @@ -33,6 +33,7 @@ public static class FirmwareHexWriter { /// one the reader rendered. An input list with no payload describes an image /// with no data, which both formats can write. /// + /// More than one non-metadata payload is supplied. public static FirmwareImage ImageFrom(IReadOnlyList inputs, string sourceFormat) { ArgumentNullException.ThrowIfNull(inputs); @@ -53,7 +54,9 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st } // Any other single input is the payload, whatever it is called: a caller // converting a .bin into a .hex should not have to rename it first. - payload ??= input.ReadContent(); + if (payload is not null) + throw new InvalidDataException($"{sourceFormat} represents one logical firmware payload; multiple payload inputs are ambiguous."); + payload = input.ReadContent(); } List<(uint Address, byte[] Data)> segments; diff --git a/docs/catalogue/our-registry.csv b/docs/catalogue/our-registry.csv index bd9da9071..cc19a97f5 100644 --- a/docs/catalogue/our-registry.csv +++ b/docs/catalogue/our-registry.csv @@ -459,6 +459,7 @@ format,Sarc,Nintendo SARC,Archive,,.sarc;.pack;.bars,Y,Y,Y,Y format,Sbc,Bluetooth SBC / mSBC,Audio,,.sbc;.msbc,Y,Y,, format,Sdat,SDAT (Nintendo DS sound archive),Audio,,.sdat,Y,Y,, format,Sds,SDS (MIDI Sample Dump),Audio,,.sds,Y,Y,, +format,SeqBox,SeqBox,Archive,,.sbx,Y,Y,Y, format,SevenZip,7z,Archive,,.7z,Y,Y,Y,Y format,Sf2,SoundFont 2,Audio,,.sf2,Y,Y,, format,Sfar,BioWare SFAR,Archive,,.sfar,Y,Y,Y, diff --git a/docs/catalogue/reference-formats.csv b/docs/catalogue/reference-formats.csv index d168a2e19..585f9f1d7 100644 --- a/docs/catalogue/reference-formats.csv +++ b/docs/catalogue/reference-formats.csv @@ -69,7 +69,7 @@ BCM,bcm,compression,,,,,R,,,1,https://github.com/encode84/bcm,Bcm,Y,Y,,OK,, Split file (.001),001;002,archive,R,,R,,R,,,3,,SplitFile,Y,Y,,OK,, Apple Archive (.aar),aar,archive,,,,,,,Y,1,,,,,,MISSING,,https://developer.apple.com/documentation/applearchive Mozilla ARchive (MAR),mar;mac,archive,,,,,,fmt/1483,Y,2,,,,,,MISSING,,https://wiki.mozilla.org/Software_Update:MAR -SeqBox,sbx,archive,,,,,,,Y,1,,,,,,MISSING,,https://github.com/MarcoPon/SeqBox +SeqBox,sbx,archive,,,,,,,Y,1,https://github.com/MarcoPon/SeqBox,SeqBox,Y,Y,,OK,, AFA,afa,archive,,,,,,,Y,1,,,,,,MISSING,, B1,b1,archive,,,,,,,Y,1,,,,,,MISSING,, B6Z,b6z,archive,,,,,,,Y,1,,,,,,MISSING,,