diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d24cac9b9..e66eb5d8f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -174,6 +174,9 @@ jobs:
# reiser4progs owns Reiser4: fsck.reiser4 and debugfs.reiser4 judge every volume the
# Reiser4 writer creates or edits, in the gate below.
sudo apt-get install -y reiser4progs
+ # zpaq (7.15) owns ZPAQ: it must list and extract what the ZPAQ writer produces, and the
+ # reader must decode every method zpaq writes, in the gate below.
+ sudo apt-get install -y zpaq
# mkbootimg is AOSP's own boot image tool pair: the Android boot image writer must
# reproduce mkbootimg's bytes, and unpack_bootimg must read back what it writes.
sudo apt-get install -y mkbootimg
@@ -243,6 +246,19 @@ jobs:
--filter "FullyQualifiedName~Compression.Tests.Reiser4.Reiser4ProfileExternalTests|FullyQualifiedName~Compression.Tests.Reiser4.Reiser4MutationExternalTests|FullyQualifiedName~Compression.Tests.Reiser4.Reiser4NativeTreeExternalTests"
--logger "trx;LogFileName=reiser4-oracle-tests.trx"
+ # Hard gate, both directions: zpaq must list and extract byte for byte what the ZPAQ writer
+ # produces (every method), the reader must decode what zpaq writes (every method, versions,
+ # deduplication, streaming), and our compressed data blocks must equal zpaq 7.15's.
+ - name: ZPAQ oracle
+ if: matrix.os == 'ubuntu-latest'
+ env:
+ CWB_REQUIRE_ZPAQ: "1"
+ run: >
+ dotnet test Compression.Tests
+ --configuration Release --no-build
+ --filter "FullyQualifiedName~Compression.Tests.Zpaq.ZpaqExternalOracleTests"
+ --logger "trx;LogFileName=zpaq-oracle-tests.trx"
+
# bcachefs-tools' own checker judges every volume this package writes or
# edits: the allocation information, the usage totals and the namespace are
# cross-examined against each other, and bucket generations against the
diff --git a/Compression.Tests/BuildingBlocks/ZpaqBuildingBlockTests.cs b/Compression.Tests/BuildingBlocks/ZpaqBuildingBlockTests.cs
index 0cf2dddf4..0ba39de84 100644
--- a/Compression.Tests/BuildingBlocks/ZpaqBuildingBlockTests.cs
+++ b/Compression.Tests/BuildingBlocks/ZpaqBuildingBlockTests.cs
@@ -33,7 +33,7 @@ public void RepeatedSentence_RoundTripsAndCompresses() {
[Test, Category("HappyPath"), Category("RoundTrip")]
public void HighlyRepetitive_RoundTripsAndCompresses() {
- var data = new byte[256];
+ var data = new byte[4096]; // large enough that the ~120-byte ZPAQ block header does not dominate
Array.Fill(data, (byte)0x61);
var compressed = Bb.Compress(data);
var round = Bb.Decompress(compressed);
diff --git a/Compression.Tests/Compression.Tests.csproj b/Compression.Tests/Compression.Tests.csproj
index 023dbbe39..d57a2c162 100644
--- a/Compression.Tests/Compression.Tests.csproj
+++ b/Compression.Tests/Compression.Tests.csproj
@@ -103,6 +103,10 @@
FreeArcVectors.%(Filename)%(Extension)
+
+
+ ZpaqVectors.%(Filename)%(Extension)
+
CvfVectors.%(Filename)%(Extension)
diff --git a/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs
new file mode 100644
index 000000000..bbbc898af
--- /dev/null
+++ b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs
@@ -0,0 +1,37 @@
+using Compression.Registry;
+using FileFormat.FirmwareHex;
+
+namespace Compression.Tests.FirmwareHex;
+
+[TestFixture]
+public sealed class FirmwareHexPayloadTests {
+ [TestCase(false)]
+ [TestCase(true)]
+ public void Create_RejectsMultiplePayloadsWithoutWritingOutput(bool emptyFirst) {
+ var descriptor = new IntelHexFormatDescriptor();
+ using var output = new MemoryStream();
+ var inputs = new[] {
+ ArchiveInputInfo.InMemory("first.bin", emptyFirst ? [] : [1, 2, 3]),
+ ArchiveInputInfo.InMemory("second.bin", [9, 8, 7]),
+ };
+
+ Assert.Throws(() =>
+ ((IArchiveCreatable)descriptor).Create(output, inputs, new FormatCreateOptions()));
+ Assert.That(output.Length, Is.Zero);
+ }
+
+ [Test]
+ public void Add_RejectsAnotherPayloadWithoutChangingTheArchive() {
+ var descriptor = new IntelHexFormatDescriptor();
+ using var archive = new MemoryStream();
+ ((IArchiveCreatable)descriptor).Create(archive,
+ [ArchiveInputInfo.InMemory("firmware.bin", [1, 2, 3])], new FormatCreateOptions());
+ var original = archive.ToArray();
+ archive.Position = 0;
+
+ Assert.Throws(() => ((IArchiveModifiable)descriptor).Add(archive,
+ [ArchiveInputInfo.InMemory("ADD.BIN", [9, 8, 7])]));
+ Assert.That(archive.ToArray(), Is.EqualTo(original));
+ }
+
+}
diff --git a/Compression.Tests/Zpaq/ReferenceVectors/README.md b/Compression.Tests/Zpaq/ReferenceVectors/README.md
new file mode 100644
index 000000000..3bec9b25a
--- /dev/null
+++ b/Compression.Tests/Zpaq/ReferenceVectors/README.md
@@ -0,0 +1,19 @@
+# zpaq 7.15 reference archives
+
+Every `.zpaq` file here was written by the reference archiver **zpaq v7.15** (Matt Mahoney,
+public domain), the Windows x64 build `zpaq64.exe` shipped in
+ ("compiled Aug 17 2016"), on 2026-10-19.
+
+`mkfixtures.py` (documentation, not a build input) generated the input files deterministically
+and ran `zpaq64.exe a [-method M]` in the work directory, so the stored names are
+relative (`set/text.txt`). Only the archives and `manifest.tsv` are checked in: the manifest gives,
+per archive and version, every name with its size and SHA-256 (size `-1` marks a directory).
+
+| File | Layout | How it was made | What it exercises |
+|------|--------|-----------------|-------------------|
+| `set.m0.zpaq` .. `set.m5.zpaq` | journaling | `a set.mN.zpaq set -method N` | every predefined method: 0 (dedup only), 1/2 (LZ77, packed codes, hash and suffix-array match finders), 3 (BWT or LZ77+CM), 4/5 (CM with ICM/ISSE chains, match model, mixers, SSE); E8E9 on `x86.bin`; `dup.txt` deduplicated against `text.txt`; an empty file; a subdirectory |
+| `versions.zpaq` | journaling | three `a versions.zpaq v` runs: add `notes.txt`+`a.bin`; change `a.bin`, add `b.txt`; delete `notes.txt` | three transactions, an update, a deletion record, fragments shared across versions |
+| `stream.zpaq` | streaming | `a stream.zpaq st -method s4.3ci1` | streaming layout (one block per file, filename and date in the segment header), BWT + order-0 ICM + order-1 ISSE |
+
+`zpaq64.exe l versions.zpaq -all` reports `3 versions, 6 files, 4 fragments`; `zpaq64.exe l
+stream.zpaq` reports two files of unknown size (the streaming layout records none).
diff --git a/Compression.Tests/Zpaq/ReferenceVectors/manifest.tsv b/Compression.Tests/Zpaq/ReferenceVectors/manifest.tsv
new file mode 100644
index 000000000..fdacb6ab4
--- /dev/null
+++ b/Compression.Tests/Zpaq/ReferenceVectors/manifest.tsv
@@ -0,0 +1,52 @@
+# archive version name size sha256 (size -1 = directory)
+set.m0.zpaq 1 set/dup.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m0.zpaq 1 set/empty.dat 0 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+set.m0.zpaq 1 set/random.bin 4096 3a370c921e1d689daae1f95c1bc8fd70c3a2294d9a602603866c88f1678133ed
+set.m0.zpaq 1 set/sub/ -1 -
+set.m0.zpaq 1 set/sub/table.bin 16384 c30bfd6c26ff4d6994184a48cfe912d634d84741a506a379a756a88d03ce17c8
+set.m0.zpaq 1 set/text.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m0.zpaq 1 set/x86.bin 16384 f80aea167fd0f16c0e8c59c394b8316b5626494ce468e4ef358e9b32cd3c05d4
+set.m1.zpaq 1 set/dup.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m1.zpaq 1 set/empty.dat 0 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+set.m1.zpaq 1 set/random.bin 4096 3a370c921e1d689daae1f95c1bc8fd70c3a2294d9a602603866c88f1678133ed
+set.m1.zpaq 1 set/sub/ -1 -
+set.m1.zpaq 1 set/sub/table.bin 16384 c30bfd6c26ff4d6994184a48cfe912d634d84741a506a379a756a88d03ce17c8
+set.m1.zpaq 1 set/text.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m1.zpaq 1 set/x86.bin 16384 f80aea167fd0f16c0e8c59c394b8316b5626494ce468e4ef358e9b32cd3c05d4
+set.m2.zpaq 1 set/dup.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m2.zpaq 1 set/empty.dat 0 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+set.m2.zpaq 1 set/random.bin 4096 3a370c921e1d689daae1f95c1bc8fd70c3a2294d9a602603866c88f1678133ed
+set.m2.zpaq 1 set/sub/ -1 -
+set.m2.zpaq 1 set/sub/table.bin 16384 c30bfd6c26ff4d6994184a48cfe912d634d84741a506a379a756a88d03ce17c8
+set.m2.zpaq 1 set/text.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m2.zpaq 1 set/x86.bin 16384 f80aea167fd0f16c0e8c59c394b8316b5626494ce468e4ef358e9b32cd3c05d4
+set.m3.zpaq 1 set/dup.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m3.zpaq 1 set/empty.dat 0 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+set.m3.zpaq 1 set/random.bin 4096 3a370c921e1d689daae1f95c1bc8fd70c3a2294d9a602603866c88f1678133ed
+set.m3.zpaq 1 set/sub/ -1 -
+set.m3.zpaq 1 set/sub/table.bin 16384 c30bfd6c26ff4d6994184a48cfe912d634d84741a506a379a756a88d03ce17c8
+set.m3.zpaq 1 set/text.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m3.zpaq 1 set/x86.bin 16384 f80aea167fd0f16c0e8c59c394b8316b5626494ce468e4ef358e9b32cd3c05d4
+set.m4.zpaq 1 set/dup.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m4.zpaq 1 set/empty.dat 0 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+set.m4.zpaq 1 set/random.bin 4096 3a370c921e1d689daae1f95c1bc8fd70c3a2294d9a602603866c88f1678133ed
+set.m4.zpaq 1 set/sub/ -1 -
+set.m4.zpaq 1 set/sub/table.bin 16384 c30bfd6c26ff4d6994184a48cfe912d634d84741a506a379a756a88d03ce17c8
+set.m4.zpaq 1 set/text.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m4.zpaq 1 set/x86.bin 16384 f80aea167fd0f16c0e8c59c394b8316b5626494ce468e4ef358e9b32cd3c05d4
+set.m5.zpaq 1 set/dup.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m5.zpaq 1 set/empty.dat 0 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+set.m5.zpaq 1 set/random.bin 4096 3a370c921e1d689daae1f95c1bc8fd70c3a2294d9a602603866c88f1678133ed
+set.m5.zpaq 1 set/sub/ -1 -
+set.m5.zpaq 1 set/sub/table.bin 16384 c30bfd6c26ff4d6994184a48cfe912d634d84741a506a379a756a88d03ce17c8
+set.m5.zpaq 1 set/text.txt 20000 6d7dcfff654e4f0b73c9c6748e20101a654d91340e808028d9cce43314e176b6
+set.m5.zpaq 1 set/x86.bin 16384 f80aea167fd0f16c0e8c59c394b8316b5626494ce468e4ef358e9b32cd3c05d4
+versions.zpaq 1 v/a.bin 3000 e17a27da77a519cad73adb40b725139379c057af5023170700519facc6e378e7
+versions.zpaq 1 v/notes.txt 6000 76c3dcfd3f6bd471dcbd9d27638a1663c5d1d0342dc1d3733d30e475922f7eff
+versions.zpaq 2 v/a.bin 3000 eaf76ffcb88a0e9add0d967f5c7c7f1918ac1c087fbf90c0c73030328d11684b
+versions.zpaq 2 v/b.txt 5000 09baec6ca8b2af5f64ce5cd913b5172d83008bc0d418dafe44b9f0f14264a121
+versions.zpaq 2 v/notes.txt 6000 76c3dcfd3f6bd471dcbd9d27638a1663c5d1d0342dc1d3733d30e475922f7eff
+versions.zpaq 3 v/a.bin 3000 eaf76ffcb88a0e9add0d967f5c7c7f1918ac1c087fbf90c0c73030328d11684b
+versions.zpaq 3 v/b.txt 5000 09baec6ca8b2af5f64ce5cd913b5172d83008bc0d418dafe44b9f0f14264a121
+stream.zpaq 1 st/one.txt 9000 fe0aff6997ba887be6c6d568be054f5600765c45a0ff3762a7c39f2d34a9f56a
+stream.zpaq 1 st/two.bin 7000 46f2d7bf41d57c5c516fad8b79c3630e264e86d8a43f22dae0dac6db793e87ba
diff --git a/Compression.Tests/Zpaq/ReferenceVectors/mkfixtures.py b/Compression.Tests/Zpaq/ReferenceVectors/mkfixtures.py
new file mode 100644
index 000000000..c12db07f0
--- /dev/null
+++ b/Compression.Tests/Zpaq/ReferenceVectors/mkfixtures.py
@@ -0,0 +1,113 @@
+"""Builds the zpaq 7.15 reference archives for Compression.Tests/Zpaq/ReferenceVectors.
+
+Usage: python -I mkfixtures.py
+The inputs are generated deterministically here; only the archives and a manifest
+(name, size, SHA-256 per version) are checked in.
+"""
+import hashlib, os, shutil, subprocess, sys
+
+zpaq, work, out = sys.argv[1], sys.argv[2], sys.argv[3]
+
+class Lcg:
+ def __init__(self, seed): self.s = seed & 0xffffffff
+ def next(self):
+ self.s = (self.s * 1103515245 + 12345) & 0xffffffff
+ return self.s >> 16
+
+WORDS = ("the of and to in is that for it as with was on be by at this which from or an are "
+ "archive journal fragment block version compress model context mixer predictor stream "
+ "zpaq level method index header data file directory size date hash checksum").split()
+
+def text(n, seed):
+ r = Lcg(seed); b = bytearray()
+ while len(b) < n:
+ w = WORDS[r.next() % len(WORDS)]
+ b += w.encode()
+ k = r.next() % 16
+ b += b'.\n' if k == 0 else (b', ' if k == 1 else b' ')
+ return bytes(b[:n])
+
+def rand(n, seed):
+ r = Lcg(seed); return bytes(r.next() & 255 for _ in range(n))
+
+def x86(n, seed):
+ r = Lcg(seed); b = bytearray()
+ while len(b) < n:
+ k = r.next() % 8
+ if k == 0: b += bytes([0xE8, r.next() & 255, r.next() & 15, 0, 0])
+ elif k == 1: b += bytes([0x8B, 0x45 + (r.next() & 7) * 8, r.next() & 0x7C])
+ elif k == 2: b += bytes([0xE9, r.next() & 255, 0xFF, 0xFF, 0xFF])
+ else: b += bytes([0x89, 0x44, 0x24, (r.next() & 15) * 4])
+ return bytes(b[:n])
+
+def table(n):
+ return bytes(((i * 7) ^ (i >> 4)) & 255 if i % 4 else (i >> 2) & 255 for i in range(n))
+
+def write(root, name, data):
+ p = os.path.join(root, name)
+ os.makedirs(os.path.dirname(p), exist_ok=True)
+ with open(p, 'wb') as f: f.write(data)
+
+def run(*args, cwd):
+ r = subprocess.run([zpaq, *args], cwd=cwd, capture_output=True, text=True)
+ if r.returncode != 0:
+ raise SystemExit(f"zpaq {' '.join(args)} failed:\n{r.stdout}\n{r.stderr}")
+
+def manifest_of(root):
+ rows = []
+ for dp, dns, fns in os.walk(root):
+ for d in dns:
+ rel = os.path.relpath(os.path.join(dp, d), os.path.dirname(root)).replace('\\', '/') + '/'
+ rows.append((rel, -1, '-'))
+ for f in fns:
+ p = os.path.join(dp, f)
+ rel = os.path.relpath(p, os.path.dirname(root)).replace('\\', '/')
+ data = open(p, 'rb').read()
+ rows.append((rel, len(data), hashlib.sha256(data).hexdigest()))
+ return sorted(rows)
+
+shutil.rmtree(work, ignore_errors=True); os.makedirs(work); os.makedirs(out, exist_ok=True)
+lines = ["# archive\tversion\tname\tsize\tsha256 (size -1 = directory)"]
+
+# One file set, every predefined method.
+s = os.path.join(work, 'set')
+t = text(20000, 1)
+write(s, 'text.txt', t)
+write(s, 'dup.txt', t) # deduplicated against text.txt
+write(s, 'x86.bin', x86(16384, 2))
+write(s, 'random.bin', rand(4096, 3))
+write(s, 'empty.dat', b'')
+write(s, 'sub/table.bin', table(16384))
+setrows = manifest_of(s)
+for m in range(6):
+ name = f'set.m{m}.zpaq'
+ run('a', name, 'set', '-method', str(m), cwd=work)
+ shutil.copy(os.path.join(work, name), out)
+ lines += [f"{name}\t1\t{n}\t{z}\t{h}" for n, z, h in setrows]
+
+# Three versions: add, modify + add, delete.
+v = os.path.join(work, 'v')
+write(v, 'notes.txt', text(6000, 4))
+write(v, 'a.bin', rand(3000, 5))
+lines += [f"versions.zpaq\t1\t{n}\t{z}\t{h}" for n, z, h in manifest_of(v)]
+run('a', 'versions.zpaq', 'v', cwd=work)
+write(v, 'a.bin', rand(3000, 6))
+write(v, 'b.txt', text(5000, 7))
+lines += [f"versions.zpaq\t2\t{n}\t{z}\t{h}" for n, z, h in manifest_of(v)]
+run('a', 'versions.zpaq', 'v', cwd=work)
+os.remove(os.path.join(v, 'notes.txt'))
+lines += [f"versions.zpaq\t3\t{n}\t{z}\t{h}" for n, z, h in manifest_of(v)]
+run('a', 'versions.zpaq', 'v', cwd=work)
+shutil.copy(os.path.join(work, 'versions.zpaq'), out)
+
+# Streaming layout (method s...): BWT + order-0/1 CM, one block per file.
+st = os.path.join(work, 'st')
+write(st, 'one.txt', text(9000, 8))
+write(st, 'two.bin', x86(7000, 9))
+run('a', 'stream.zpaq', 'st', '-method', 's4.3ci1', cwd=work)
+shutil.copy(os.path.join(work, 'stream.zpaq'), out)
+lines += [f"stream.zpaq\t1\t{n}\t{z}\t{h}" for n, z, h in manifest_of(st) if z >= 0]
+
+with open(os.path.join(out, 'manifest.tsv'), 'w', newline='\n') as f:
+ f.write('\n'.join(lines) + '\n')
+print('ok')
diff --git a/Compression.Tests/Zpaq/ReferenceVectors/set.m0.zpaq b/Compression.Tests/Zpaq/ReferenceVectors/set.m0.zpaq
new file mode 100644
index 000000000..31fa6539c
Binary files /dev/null and b/Compression.Tests/Zpaq/ReferenceVectors/set.m0.zpaq differ
diff --git a/Compression.Tests/Zpaq/ReferenceVectors/set.m1.zpaq b/Compression.Tests/Zpaq/ReferenceVectors/set.m1.zpaq
new file mode 100644
index 000000000..6df68c7e8
Binary files /dev/null and b/Compression.Tests/Zpaq/ReferenceVectors/set.m1.zpaq differ
diff --git a/Compression.Tests/Zpaq/ReferenceVectors/set.m2.zpaq b/Compression.Tests/Zpaq/ReferenceVectors/set.m2.zpaq
new file mode 100644
index 000000000..1d7a3cf40
Binary files /dev/null and b/Compression.Tests/Zpaq/ReferenceVectors/set.m2.zpaq differ
diff --git a/Compression.Tests/Zpaq/ReferenceVectors/set.m3.zpaq b/Compression.Tests/Zpaq/ReferenceVectors/set.m3.zpaq
new file mode 100644
index 000000000..7e1c441dc
Binary files /dev/null and b/Compression.Tests/Zpaq/ReferenceVectors/set.m3.zpaq differ
diff --git a/Compression.Tests/Zpaq/ReferenceVectors/set.m4.zpaq b/Compression.Tests/Zpaq/ReferenceVectors/set.m4.zpaq
new file mode 100644
index 000000000..2d416ee1e
Binary files /dev/null and b/Compression.Tests/Zpaq/ReferenceVectors/set.m4.zpaq differ
diff --git a/Compression.Tests/Zpaq/ReferenceVectors/set.m5.zpaq b/Compression.Tests/Zpaq/ReferenceVectors/set.m5.zpaq
new file mode 100644
index 000000000..74ba0d2aa
Binary files /dev/null and b/Compression.Tests/Zpaq/ReferenceVectors/set.m5.zpaq differ
diff --git a/Compression.Tests/Zpaq/ReferenceVectors/stream.zpaq b/Compression.Tests/Zpaq/ReferenceVectors/stream.zpaq
new file mode 100644
index 000000000..196fb551b
Binary files /dev/null and b/Compression.Tests/Zpaq/ReferenceVectors/stream.zpaq differ
diff --git a/Compression.Tests/Zpaq/ReferenceVectors/versions.zpaq b/Compression.Tests/Zpaq/ReferenceVectors/versions.zpaq
new file mode 100644
index 000000000..067707d49
Binary files /dev/null and b/Compression.Tests/Zpaq/ReferenceVectors/versions.zpaq differ
diff --git a/Compression.Tests/Zpaq/ZpaqEngineTests.cs b/Compression.Tests/Zpaq/ZpaqEngineTests.cs
new file mode 100644
index 000000000..42586961f
--- /dev/null
+++ b/Compression.Tests/Zpaq/ZpaqEngineTests.cs
@@ -0,0 +1,128 @@
+using System.Text;
+using Compression.Tests.Csc;
+using FileFormat.Zpaq;
+
+namespace Compression.Tests.Zpaq;
+
+///
+/// The libzpaq conversion below the archive layer: suffix arrays, the ZPAQL compiler and
+/// interpreter, method expansion, and single-block streams.
+///
+[TestFixture]
+public class ZpaqEngineTests {
+
+ // ── suffix array ─────────────────────────────────────────────────────────
+
+ private static int[] NaiveSuffixArray(byte[] s) =>
+ Enumerable.Range(0, s.Length).OrderBy(i => i, Comparer.Create((a, b) => s.AsSpan(a).SequenceCompareTo(s.AsSpan(b)))).ToArray();
+
+ [Test, Category("HappyPath")]
+ public void SuffixArray_MatchesNaiveSort([Values(0, 1, 2, 3, 17, 100, 1000)] int length, [Values(2, 4, 256)] int alphabet) {
+ var rng = new Random(length * 31 + alphabet);
+ var s = new byte[length];
+ for (var i = 0; i < s.Length; ++i)
+ s[i] = (byte)rng.Next(alphabet);
+ Assert.That(ZpaqSuffixArray.Build(s), Is.EqualTo(NaiveSuffixArray(s)));
+ }
+
+ [Test, Category("BoundaryCase")]
+ public void SuffixArray_PeriodicAndConstantInputs() {
+ foreach (var s in new[] { Enumerable.Repeat((byte)7, 500).ToArray(), Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat("abcab", 97))), Encoding.ASCII.GetBytes("mississippi") })
+ Assert.That(ZpaqSuffixArray.Build(s), Is.EqualTo(NaiveSuffixArray(s)));
+ }
+
+ // ── squash / stretch (specification section 7, checked by libzpaq's own self test) ──
+
+ [Test, Category("HappyPath")]
+ public void SquashAndStretch_MatchLibzpaqChecksums() {
+ uint stsum = 0, sqsum = 0;
+ for (var i = 32767; i >= 0; --i)
+ stsum = stsum * 3 + (uint)ZpaqPredictor.Stretch(i);
+ for (var i = 4095; i >= 0; --i)
+ sqsum = sqsum * 3 + (uint)ZpaqPredictor.Squash(i - 2048);
+ Assert.Multiple(() => {
+ Assert.That(stsum, Is.EqualTo(3887533746u));
+ Assert.That(sqsum, Is.EqualTo(2278286169u));
+ });
+ }
+
+ // ── compiler ─────────────────────────────────────────────────────────────
+
+ [Test, Category("HappyPath")]
+ public void Compiler_ProducesLibzpaqMinCfgBytecode() {
+ // min.cfg from the libzpaq documentation; the expected bytes are libzpaq's built-in "Model 1".
+ const string minCfg = "comp 1 2 0 0 2 0 icm 16 1 isse 19 0 hcomp *b=a a=0 d=0 hash b-- hash *d=a d++ b-- hash b-- hash *d=a halt post 0 end";
+ var hz = new ZpaqlMachine();
+ var pz = new ZpaqlMachine();
+ _ = new ZpaqCompiler(minCfg, null, hz, pz);
+ var ms = new ZpaqBuffer();
+ hz.Write(ms, false);
+ byte[] model1 = [26, 0, 1, 2, 0, 0, 2, 3, 16, 8, 19, 0, 0, 96, 4, 28, 59, 10, 59, 112, 25, 10, 59, 10, 59, 112, 56, 0];
+ Assert.That(ms.ToArray(), Is.EqualTo(model1));
+ }
+
+ [Test, Category("Exception")]
+ public void Compiler_RejectsBadConfigs([Values("comp 0 0 0 0 1 0 bogus hcomp halt end", "comp 0 0 0 0 0 hcomp endif halt end", "comp 0 0 0 0 0 hcomp halt")] string config) =>
+ Assert.That(() => new ZpaqCompiler(config, null, new(), new()), Throws.InstanceOf());
+
+ [Test, Category("HappyPath")]
+ public void MakeConfig_ExpandsArguments() {
+ var args = new int[9];
+ var config = ZpaqMethods.MakeConfig("x6.3ci1", args);
+ Assert.Multiple(() => {
+ Assert.That(args[..2], Is.EqualTo(new[] { 6, 3 }));
+ Assert.That(config, Does.StartWith("comp 9 16 $1+20 $1+20 2\n0 icm"));
+ Assert.That(config, Does.Contain("pcomp bwtrle c ;"));
+ Assert.That(ZpaqMethods.MakeConfig("0", args), Is.EqualTo("comp 0 0 0 0 0 hcomp end\n"));
+ });
+ }
+
+ // ── single-block streams ─────────────────────────────────────────────────
+
+ private static byte[] RoundTrip(byte[] data, string method) {
+ var output = new ZpaqBuffer();
+ ZpaqMethods.CompressBlock(new ZpaqBuffer(data), output, method, "name", null, storeSha1: true);
+ return ZpaqStream.Decompress(output.Span);
+ }
+
+ [Test, Category("RoundTrip")]
+ public void Block_EveryTransformAndModel_RoundTrips([Values("0", "x0.0c0.0.255", "x0.1.4.0.3.20", "x0.2.4.0.3.20c0.0.511", "x0.3ci1", "x0.4", "x0.5.4.0.3.20", "x0.6.4.0.3.20", "x0.7ci1",
+ "x0.1.4.0.5.21", "x0.0c256.0.255.1511.255i1aw1m16s8.32.255", "x0.0c0.1010.255t", "x0.0ci1,1,1,1,2aw2mm16tst")] string method) {
+ byte[] data = [.. CscSamples.Text(9000, 31), .. CscSamples.X86(7000, 32), .. CscSamples.RandomBytes(2000, 33)];
+ Assert.That(RoundTrip(data, method), Is.EqualTo(data));
+ }
+
+ [Test, Category("RoundTrip")]
+ public void Block_LevelsWithHints_RoundTrip([Values("1", "2", "3", "4", "5")] string level, [Values(",0,0", ",200,1", ",120,2", ",250,3")] string hints) {
+ var data = CscSamples.Mixed();
+ Assert.That(RoundTrip(data, level + "0" + hints), Is.EqualTo(data));
+ }
+
+ [Test, Category("BoundaryCase")]
+ public void Block_EmptyAndOneByte_RoundTrip([Values("0", "x0.1.4.0.3.20", "x0.2.4.0.3.20", "x0.3ci1", "x0.0ci1")] string method) {
+ Assert.Multiple(() => {
+ Assert.That(RoundTrip([], method), Is.Empty);
+ Assert.That(RoundTrip([42], method), Is.EqualTo(new byte[] { 42 }));
+ });
+ }
+
+ [Test, Category("Exception")]
+ public void Block_FlippedChecksum_IsDetected() {
+ var output = new ZpaqBuffer();
+ ZpaqMethods.CompressBlock(new ZpaqBuffer(CscSamples.Text(5000, 34)), output, "x0.0ci1", null, null, storeSha1: true);
+ var bytes = output.ToArray();
+ bytes[^3] ^= 1; // inside the SHA-1 trailer, before the end-of-block byte
+ Assert.That(() => ZpaqStream.Decompress(bytes), Throws.InstanceOf());
+ }
+
+ [Test, Category("HappyPath")]
+ public void E8E9_IsUndoneByThePostprocessor() {
+ var data = CscSamples.X86(30_000, 35);
+ var copy = (byte[])data.Clone();
+ ZpaqMethods.E8E9(copy);
+ Assert.Multiple(() => {
+ Assert.That(copy, Is.Not.EqualTo(data), "the transform changes call targets");
+ Assert.That(RoundTrip(data, "x0.4"), Is.EqualTo(data));
+ });
+ }
+}
diff --git a/Compression.Tests/Zpaq/ZpaqExternalOracleTests.cs b/Compression.Tests/Zpaq/ZpaqExternalOracleTests.cs
new file mode 100644
index 000000000..8de4b060b
--- /dev/null
+++ b/Compression.Tests/Zpaq/ZpaqExternalOracleTests.cs
@@ -0,0 +1,252 @@
+using System.Diagnostics;
+using System.Text;
+using Compression.Tests.Csc;
+using FileFormat.Zpaq;
+
+namespace Compression.Tests.Zpaq;
+
+///
+/// Live checks against the reference archiver zpaq 7.15, both directions: zpaq must list and
+/// extract byte for byte what writes, we must read what zpaq writes (every
+/// method, several versions, deduplication, streaming), and with the same input our compressed data
+/// blocks must equal zpaq's.
+///
+///
+/// The tool is $ZPAQ (a path), else zpaq on the host PATH, else zpaq inside WSL.
+/// Debian and Ubuntu package zpaq 7.15 (apt-get install zpaq); the Windows build is
+/// zpaq64.exe from . Without the tool every
+/// test is ignored, unless CWB_REQUIRE_ZPAQ=1 (set by the CI oracle step) turns that into a failure.
+///
+[TestFixture]
+[Category("ExternalInterop")]
+public sealed class ZpaqExternalOracleTests {
+ private static readonly Lazy?> _tool = new(LocateTool);
+ private static readonly DateTime When = new(2026, 10, 19, 12, 0, 0, DateTimeKind.Utc);
+ private string _tmp = null!;
+
+ [SetUp]
+ public void Setup() {
+ this._tmp = Path.Combine(Path.GetTempPath(), $"cwb_zpaq_{Guid.NewGuid():N}");
+ Directory.CreateDirectory(this._tmp);
+ }
+
+ [TearDown]
+ public void Teardown() {
+ try { Directory.Delete(this._tmp, recursive: true); } catch { /* best effort */ }
+ }
+
+ private static (string Name, byte[] Data)[] Sample() {
+ var text = CscSamples.Text(40_000, 41);
+ return [
+ ("set/text.txt", text),
+ ("set/dup.txt", text),
+ ("set/x86.bin", CscSamples.X86(30_000, 42)),
+ ("set/random.bin", CscSamples.RandomBytes(12_000, 43)),
+ ("set/table.bin", CscSamples.Table(20_000)),
+ ("set/empty.dat", []),
+ ("set/sub/mixed.bin", CscSamples.Mixed()),
+ ("set/sub/big.bin", [.. CscSamples.RandomBytes(700_000, 44), .. CscSamples.Text(300_000, 45)]),
+ ];
+ }
+
+ private void Materialise(string root, (string Name, byte[] Data)[] files) {
+ foreach (var (name, data) in files) {
+ var path = Path.Combine(root, name.Replace('/', Path.DirectorySeparatorChar));
+ Directory.CreateDirectory(Path.GetDirectoryName(path)!);
+ File.WriteAllBytes(path, data);
+ }
+ }
+
+ private static byte[] WriteOurs(string method, (string Name, byte[] Data)[] files) {
+ using var ms = new MemoryStream();
+ using (var w = new ZpaqWriter(ms, leaveOpen: true, method) { TransactionDate = When }) {
+ w.AddDirectory("set/");
+ w.AddDirectory("set/sub/");
+ foreach (var (name, data) in files)
+ w.AddFile(name, data);
+ }
+ return ms.ToArray();
+ }
+
+ public static IEnumerable Methods() => ["0", "1", "2", "3", "4", "5", "14", "36", "x4.0ci1.1.1.1.2aw2mm16tst", "x4.3ci1"];
+
+ // ── zpaq reads us ────────────────────────────────────────────────────────
+
+ [Test, TestCaseSource(nameof(Methods))]
+ public void OurArchive_IsListedAndExtractedByZpaq(string method) {
+ var tool = RequireTool();
+ var files = Sample();
+ File.WriteAllBytes(Path.Combine(this._tmp, "ours.zpaq"), WriteOurs(method, files));
+
+ var list = tool(this._tmp, ["l", "ours.zpaq"]);
+ var extract = tool(this._tmp, ["x", "ours.zpaq", "-to", "out"]);
+
+ Assert.Multiple(() => {
+ Assert.That(list.ExitCode, Is.Zero, list.Output);
+ Assert.That(list.Output, Does.Contain("1 versions, 10 files"), list.Output);
+ Assert.That(extract.ExitCode, Is.Zero, extract.Output);
+ foreach (var (name, data) in files)
+ Assert.That(File.ReadAllBytes(Path.Combine(this._tmp, "out", name.Replace('/', Path.DirectorySeparatorChar))), Is.EqualTo(data), name);
+ Assert.That(Directory.Exists(Path.Combine(this._tmp, "out", "set", "sub")), "directory entry restored");
+ });
+ }
+
+ // ── we read zpaq ─────────────────────────────────────────────────────────
+
+ [Test, TestCaseSource(nameof(Methods))]
+ public void ZpaqArchive_IsReadByUs(string method) {
+ var tool = RequireTool();
+ var files = Sample();
+ this.Materialise(this._tmp, files);
+ var add = tool(this._tmp, ["a", "theirs.zpaq", "set", "-method", method]);
+ Assert.That(add.ExitCode, Is.Zero, add.Output);
+
+ using var r = new ZpaqReader(File.OpenRead(Path.Combine(this._tmp, "theirs.zpaq")));
+ var entries = r.Entries.ToDictionary(e => e.FileName);
+ Assert.Multiple(() => {
+ Assert.That(r.Errors, Is.Zero);
+ Assert.That(entries.Keys, Is.SupersetOf(new[] { "set/", "set/sub/" }));
+ foreach (var (name, data) in files)
+ Assert.That(r.ExtractBytes(entries[name]), Is.EqualTo(data), name);
+ });
+ }
+
+ [Test]
+ public void ZpaqVersions_UpdatesAndDeletions_AreReadByUs() {
+ var tool = RequireTool();
+ var root = this._tmp;
+ this.Materialise(root, [("v/a.txt", CscSamples.Text(9000, 46)), ("v/b.bin", CscSamples.RandomBytes(5000, 47))]);
+ Assert.That(tool(root, ["a", "v.zpaq", "v"]).ExitCode, Is.Zero);
+ Thread.Sleep(1100); // zpaq versions need distinct second-resolution dates
+ this.Materialise(root, [("v/b.bin", CscSamples.RandomBytes(5000, 48)), ("v/c.txt", CscSamples.Text(4000, 49))]);
+ Assert.That(tool(root, ["a", "v.zpaq", "v", "-method", "3"]).ExitCode, Is.Zero);
+ Thread.Sleep(1100);
+ File.Delete(Path.Combine(root, "v", "a.txt"));
+ Assert.That(tool(root, ["a", "v.zpaq", "v", "-method", "5"]).ExitCode, Is.Zero);
+
+ using var r = new ZpaqReader(File.OpenRead(Path.Combine(root, "v.zpaq")));
+ var v1 = r.GetEntries(1).ToDictionary(e => e.FileName);
+ var v3 = r.Entries.ToDictionary(e => e.FileName);
+ Assert.Multiple(() => {
+ Assert.That(r.VersionCount, Is.EqualTo(3));
+ Assert.That(r.ExtractBytes(v1["v/b.bin"]), Is.EqualTo(CscSamples.RandomBytes(5000, 47)));
+ Assert.That(r.ExtractBytes(v1["v/a.txt"]), Is.EqualTo(CscSamples.Text(9000, 46)));
+ Assert.That(r.ExtractBytes(v3["v/b.bin"]), Is.EqualTo(CscSamples.RandomBytes(5000, 48)));
+ Assert.That(r.ExtractBytes(v3["v/c.txt"]), Is.EqualTo(CscSamples.Text(4000, 49)));
+ Assert.That(v3.ContainsKey("v/a.txt"), Is.False, "deleted in version 3");
+ });
+ }
+
+ [Test]
+ public void ZpaqStreamingArchive_IsReadByUs([Values("s4.3ci1", "s1", "s0.0c0.0.255.255i2")] string method) {
+ var tool = RequireTool();
+ var files = Sample().Where(f => f.Data.Length > 0).ToArray();
+ this.Materialise(this._tmp, files);
+ Assert.That(tool(this._tmp, ["a", "s.zpaq", "set", "-method", method]).ExitCode, Is.Zero);
+
+ using var r = new ZpaqReader(File.OpenRead(Path.Combine(this._tmp, "s.zpaq")));
+ var entries = r.Entries.ToDictionary(e => e.FileName);
+ Assert.Multiple(() => {
+ foreach (var (name, data) in files)
+ Assert.That(r.ExtractBytes(entries[name]), Is.EqualTo(data), name);
+ });
+ }
+
+ [Test]
+ public void OurBuildingBlockStream_IsExtractedByZpaq() {
+ var tool = RequireTool();
+ var data = CscSamples.Mixed();
+ File.WriteAllBytes(Path.Combine(this._tmp, "bb.zpaq"), new ZpaqBuildingBlock().Compress(data));
+ var extract = tool(this._tmp, ["x", "bb.zpaq", "-to", "out"]);
+ Assert.That(extract.ExitCode, Is.Zero, extract.Output);
+ var produced = Directory.GetFiles(Path.Combine(this._tmp, "out"), "*", SearchOption.AllDirectories);
+ Assert.That(produced.Select(File.ReadAllBytes), Has.Some.EqualTo(data));
+ }
+
+ // ── byte equality ────────────────────────────────────────────────────────
+
+ [Test]
+ public void OurDataBlocks_EqualZpaqs([Values("0", "1", "2", "3", "4", "5")] string method) {
+ var tool = RequireTool();
+ var banner = tool(this._tmp, []).Output;
+ if (!banner.Contains("v7.15"))
+ Assert.Ignore("Data-block equality is pinned against zpaq v7.15; found: " + banner.Split('\n')[0]);
+ var files = Sample();
+ this.Materialise(this._tmp, files);
+ Assert.That(tool(this._tmp, ["a", "theirs.zpaq", "set", "-method", method]).ExitCode, Is.Zero);
+ var theirs = File.ReadAllBytes(Path.Combine(this._tmp, "theirs.zpaq"));
+ var ours = WriteOurs(method, files);
+
+ Assert.That(DataRegion(ours), Is.EqualTo(DataRegion(theirs)), "d blocks with the transaction date masked");
+ }
+
+ /// The bytes from the first d block to the first h block, with every journaling date zeroed.
+ private static byte[] DataRegion(byte[] archive) {
+ var s = Encoding.Latin1.GetString(archive);
+ var d = JournalName(s, 'd');
+ var h = JournalName(s, 'h');
+ if (d < 0)
+ return [];
+ var tag = Encoding.Latin1.GetString(ZpaqCompressorTag);
+ var start = s.LastIndexOf(tag, d, StringComparison.Ordinal);
+ var end = s.LastIndexOf(tag, h, StringComparison.Ordinal);
+ var region = archive[start..end];
+ var text = Encoding.Latin1.GetString(region);
+ for (var i = text.IndexOf("jDC", StringComparison.Ordinal); i >= 0; i = text.IndexOf("jDC", i + 3, StringComparison.Ordinal))
+ if (i + 17 <= region.Length && text.AsSpan(i + 3, 14).IndexOfAnyExceptInRange('0', '9') < 0)
+ region.AsSpan(i + 3, 14).Fill((byte)'0');
+ return region;
+ }
+
+ private static int JournalName(string s, char type) {
+ for (var i = s.IndexOf("jDC", StringComparison.Ordinal); i >= 0; i = s.IndexOf("jDC", i + 3, StringComparison.Ordinal))
+ if (i + 18 <= s.Length && s[i + 17] == type && s.AsSpan(i + 3, 14).IndexOfAnyExceptInRange('0', '9') < 0)
+ return i;
+ return -1;
+ }
+
+ private static readonly byte[] ZpaqCompressorTag = [0x37, 0x6b, 0x53, 0x74, 0xa0, 0x31, 0x83, 0xd3, 0x8c, 0xb2, 0x28, 0xb0, 0xd3];
+
+ // ── tool location ────────────────────────────────────────────────────────
+
+ private static Func RequireTool() {
+ var tool = _tool.Value;
+ if (tool is null) {
+ if (Environment.GetEnvironmentVariable("CWB_REQUIRE_ZPAQ") == "1")
+ Assert.Fail("CWB_REQUIRE_ZPAQ=1 but zpaq was not found ($ZPAQ, PATH, WSL).");
+ Assert.Ignore("zpaq not found: set ZPAQ or install zpaq 7.15 (see the class remarks).");
+ }
+ return tool!;
+ }
+
+ private static Func? LocateTool() {
+ var configured = Environment.GetEnvironmentVariable("ZPAQ");
+ if (!string.IsNullOrWhiteSpace(configured) && File.Exists(configured))
+ return (dir, args) => Run(configured, dir, args);
+ if (FsInteropToolbox.HostPathHasTool("zpaq"))
+ return (dir, args) => Run("zpaq", dir, args);
+ if (OperatingSystem.IsWindows() && FsInteropToolbox.WslAvailable && FsInteropToolbox.WslHasTool("zpaq"))
+ return (dir, args) => {
+ var line = string.Join(' ', args.Select(a => "'" + a.Replace("'", "'\\''") + "'"));
+ var (stdOut, stdErr, exitCode) = FsInteropToolbox.RunWsl($"cd '{FsInteropToolbox.WinToWsl(dir)}' && zpaq {line}");
+ return (exitCode, stdOut + stdErr);
+ };
+ return null;
+ }
+
+ private static (int ExitCode, string Output) Run(string executable, string workingDirectory, string[] arguments) {
+ var start = new ProcessStartInfo(executable) {
+ WorkingDirectory = workingDirectory,
+ RedirectStandardOutput = true,
+ RedirectStandardError = true,
+ UseShellExecute = false,
+ };
+ foreach (var argument in arguments)
+ start.ArgumentList.Add(argument);
+ using var process = Process.Start(start)!;
+ var stdOut = process.StandardOutput.ReadToEndAsync();
+ var stdErr = process.StandardError.ReadToEndAsync();
+ process.WaitForExit();
+ return (process.ExitCode, stdOut.Result + stdErr.Result);
+ }
+}
diff --git a/Compression.Tests/Zpaq/ZpaqRangeCoderTests.cs b/Compression.Tests/Zpaq/ZpaqRangeCoderTests.cs
deleted file mode 100644
index 8d03a7a5b..000000000
--- a/Compression.Tests/Zpaq/ZpaqRangeCoderTests.cs
+++ /dev/null
@@ -1,223 +0,0 @@
-using FileFormat.Zpaq;
-
-namespace Compression.Tests.Zpaq;
-
-///
-/// Pins the safety invariant of the ZPAQ binary range coder: on entry to every bit
-/// the range lies in [2^24, 2^32) and the probability in [1, 65535], and both
-/// subranges are then at least 256 wide, so the interval can never collapse to
-/// zero width and leave the coder unable to distinguish the two bits.
-///
-[TestFixture]
-public class ZpaqRangeCoderTests {
-
- private const uint RangeMin = ZpaqRangeCoder.RangeMinimum;
- private const uint MinimumSubrangeWidth = 256;
-
- /// Ranges that bracket every interesting case of the split arithmetic.
- private static IEnumerable InterestingRanges() {
- yield return RangeMin; // smallest range the invariant allows
- yield return RangeMin + 1;
- yield return RangeMin * 2;
- yield return 0x7FFFFFFFu;
- yield return 0x80000000u;
- yield return 0xFFFF0000u;
- yield return 0xFFFFFFFFu; // largest range, the initial one
- var rng = new Random(0x5EED);
- for (var i = 0; i < 512; ++i)
- yield return RangeMin + (uint)rng.NextInt64(0, 0xFFFFFFFFL - RangeMin + 1);
- }
-
- /// Probabilities that bracket every interesting case of the split arithmetic.
- private static IEnumerable InterestingProbabilities() {
- yield return ZpaqRangeCoder.MinimumProbability;
- yield return 2;
- yield return 255;
- yield return 256;
- yield return 32768;
- yield return 65534;
- yield return ZpaqRangeCoder.MaximumProbability;
- var rng = new Random(0xC0FFEE);
- for (var i = 0; i < 64; ++i)
- yield return rng.Next(1, 65536);
- }
-
- // ── The invariant itself ─────────────────────────────────────────────────
-
- [Test, Category("BoundaryCase")]
- public void Split_LeavesBothSubrangesAtLeast256Wide() {
- foreach (var range in InterestingRanges())
- foreach (var probability in InterestingProbabilities()) {
- var bound = ZpaqRangeCoder.Split(range, probability);
- Assert.That(
- bound,
- Is.GreaterThanOrEqualTo(MinimumSubrangeWidth),
- $"the 1 subrange collapsed at range={range}, p={probability}");
- Assert.That(
- range - bound,
- Is.GreaterThanOrEqualTo(MinimumSubrangeWidth),
- $"the 0 subrange collapsed at range={range}, p={probability}");
- Assert.That(bound, Is.LessThan(range));
- }
- }
-
- [Test, Category("EdgeCase")]
- public void Split_ClampsProbabilitiesOutsideTheSafeBand() {
- // A model that produced 0 or 65536 would otherwise put the split on an end of
- // the interval. The clamp is what makes the invariant hold unconditionally.
- foreach (var range in new uint[] { RangeMin, 0xFFFFFFFFu }) {
- Assert.That(ZpaqRangeCoder.Split(range, 0), Is.EqualTo(ZpaqRangeCoder.Split(range, 1)));
- Assert.That(ZpaqRangeCoder.Split(range, -1000), Is.EqualTo(ZpaqRangeCoder.Split(range, 1)));
- Assert.That(ZpaqRangeCoder.Split(range, 65536), Is.EqualTo(ZpaqRangeCoder.Split(range, 65535)));
- Assert.That(ZpaqRangeCoder.Split(range, int.MaxValue), Is.EqualTo(ZpaqRangeCoder.Split(range, 65535)));
- }
- }
-
- [Test, Category("BoundaryCase")]
- public void Encoder_KeepsTheRangeNormalisedAfterEveryBit() {
- // Coding the least likely bit every time is the fastest way to shrink the
- // range, so it is the case that would expose a renormalisation that does not
- // actually restore the invariant.
- using var output = new MemoryStream();
- var coder = new ZpaqRangeEncoder(output);
- for (var i = 0; i < 20000; ++i) {
- var probability = i % 2 == 0 ? 1 : 65535;
- var bit = i % 2 == 0 ? 1 : 0; // always the improbable one
- coder.EncodeBit(bit, probability);
- Assert.That(coder.Range, Is.GreaterThanOrEqualTo(RangeMin), $"range collapsed after bit {i}");
- }
- coder.Flush();
- }
-
- [Test, Category("BoundaryCase")]
- public void Decoder_KeepsTheRangeNormalisedAndReproducesEveryBit() {
- var bits = new int[20000];
- var probabilities = new int[bits.Length];
- var rng = new Random(0xBEEF);
- for (var i = 0; i < bits.Length; ++i) {
- // Weight heavily towards the extremes, where an interval collapse would show.
- probabilities[i] = (i % 3) switch {
- 0 => 1,
- 1 => 65535,
- _ => rng.Next(1, 65536),
- };
- bits[i] = rng.Next(2);
- }
-
- using var output = new MemoryStream();
- var encoder = new ZpaqRangeEncoder(output);
- for (var i = 0; i < bits.Length; ++i)
- encoder.EncodeBit(bits[i], probabilities[i]);
- encoder.Flush();
-
- var decoder = new ZpaqRangeDecoder(output.ToArray(), 0);
- for (var i = 0; i < bits.Length; ++i) {
- var bit = decoder.DecodeBit(probabilities[i]);
- Assert.That(bit, Is.EqualTo(bits[i]), $"bit {i} decoded wrongly");
- Assert.That(decoder.Range, Is.GreaterThanOrEqualTo(RangeMin), $"range collapsed after bit {i}");
- }
- }
-
- // ── Coding behaviour ─────────────────────────────────────────────────────
-
- [Test, Category("HappyPath")]
- public void ConfidentPredictionsCostAlmostNothing() {
- // The lower subrange belongs to a 1 bit, so a confident 1 must be cheap. If the
- // convention were reversed the stream would grow instead of shrinking.
- using var output = new MemoryStream();
- var coder = new ZpaqRangeEncoder(output);
- for (var i = 0; i < 100000; ++i)
- coder.EncodeBit(1, 65535);
- coder.Flush();
- Assert.That(output.Length, Is.LessThan(100000 / 8));
- }
-
- [Test, Category("HappyPath")]
- public void RandomBitsAtOneHalfCostAboutOneBitEach() {
- const int Count = 80000;
- var rng = new Random(7);
- var bits = new int[Count];
- for (var i = 0; i < Count; ++i)
- bits[i] = rng.Next(2);
-
- using var output = new MemoryStream();
- var encoder = new ZpaqRangeEncoder(output);
- foreach (var bit in bits)
- encoder.EncodeBit(bit, 32768);
- encoder.Flush();
-
- Assert.That(output.Length, Is.InRange(Count / 8, Count / 8 + 32));
-
- var decoder = new ZpaqRangeDecoder(output.ToArray(), 0);
- foreach (var bit in bits)
- Assert.That(decoder.DecodeBit(32768), Is.EqualTo(bit));
- }
-
- [Test, Category("BoundaryCase")]
- public void Flush_ResolvesEveryDeferredByteAcrossAllTailStates() {
- // The flush is the subtlest part of a carry-counting coder: when the last bytes
- // of low are 0xFF they are only counted, not written, and they must still all
- // come out. Short messages with extreme probabilities are what put the coder
- // into those tail states, so sweep many of them rather than reasoning about one.
- var rng = new Random(0xF1005);
- for (var trial = 0; trial < 5000; ++trial) {
- var count = rng.Next(1, 65);
- var bits = new int[count];
- var probabilities = new int[count];
- for (var i = 0; i < count; ++i) {
- probabilities[i] = rng.Next(4) switch {
- 0 => 1,
- 1 => 65535,
- 2 => 32768,
- _ => rng.Next(1, 65536),
- };
- bits[i] = rng.Next(2);
- }
-
- using var output = new MemoryStream();
- var encoder = new ZpaqRangeEncoder(output);
- for (var i = 0; i < count; ++i)
- encoder.EncodeBit(bits[i], probabilities[i]);
- encoder.Flush();
-
- var decoder = new ZpaqRangeDecoder(output.ToArray(), 0);
- for (var i = 0; i < count; ++i)
- Assert.That(
- decoder.DecodeBit(probabilities[i]),
- Is.EqualTo(bits[i]),
- $"trial {trial}, bit {i} of {count} did not survive the flush");
- }
- }
-
- [Test, Category("EdgeCase")]
- public void EmptyStream_FlushesFiveBytesAndTheFirstIsTheEmptyCache() {
- using var output = new MemoryStream();
- var coder = new ZpaqRangeEncoder(output);
- coder.Flush();
- var bytes = output.ToArray();
- Assert.That(bytes, Has.Length.EqualTo(ZpaqRangeCoder.FlushBytes));
- Assert.That(bytes[0], Is.Zero);
- }
-
- // ── Exceptional input ────────────────────────────────────────────────────
-
- [Test, Category("Exception")]
- public void Encoder_NullStream_Throws() =>
- Assert.That(() => new ZpaqRangeEncoder(null!), Throws.ArgumentNullException);
-
- [Test, Category("Exception")]
- public void Decoder_NullBuffer_Throws() =>
- Assert.That(() => new ZpaqRangeDecoder(null!, 0), Throws.ArgumentNullException);
-
- [Test, Category("EdgeCase")]
- public void Decoder_TruncatedStream_PadsWithZeroesInsteadOfFailing() {
- // A decoder that ran off the end must stay deterministic rather than throw,
- // because the last few bits of a message are pinned by the flush.
- var decoder = new ZpaqRangeDecoder([], 0);
- Assert.That(() => {
- for (var i = 0; i < 1000; ++i)
- decoder.DecodeBit(32768);
- }, Throws.Nothing);
- Assert.That(decoder.Range, Is.GreaterThanOrEqualTo(RangeMin));
- }
-}
diff --git a/Compression.Tests/Zpaq/ZpaqReferenceVectorTests.cs b/Compression.Tests/Zpaq/ZpaqReferenceVectorTests.cs
new file mode 100644
index 000000000..8a05c0bee
--- /dev/null
+++ b/Compression.Tests/Zpaq/ZpaqReferenceVectorTests.cs
@@ -0,0 +1,111 @@
+using System.Reflection;
+using System.Security.Cryptography;
+using FileFormat.Zpaq;
+
+namespace Compression.Tests.Zpaq;
+
+///
+/// Archives written by the reference zpaq 7.15 (see ReferenceVectors/README.md): every version of
+/// every archive must list and extract to exactly what manifest.tsv records.
+///
+[TestFixture]
+public sealed class ZpaqReferenceVectorTests {
+ private sealed record Row(string Archive, int Version, string Name, long Size, string Sha256);
+
+ private static readonly Lazy Manifest = new(() => {
+ using var reader = new StreamReader(Open("manifest.tsv"));
+ return reader.ReadToEnd().Split('\n', StringSplitOptions.RemoveEmptyEntries)
+ .Select(l => l.TrimEnd('\r'))
+ .Where(l => l.Length > 0 && !l.StartsWith('#'))
+ .Select(l => l.Split('\t'))
+ .Select(f => new Row(f[0], int.Parse(f[1]), f[2], long.Parse(f[3]), f[4]))
+ .ToArray();
+ });
+
+ private static Stream Open(string name) =>
+ Assembly.GetExecutingAssembly().GetManifestResourceStream($"ZpaqVectors.{name}")
+ ?? throw new FileNotFoundException($"Embedded zpaq vector '{name}' is missing from the test assembly.");
+
+ private static byte[] Load(string name) {
+ using var s = Open(name);
+ using var ms = new MemoryStream();
+ s.CopyTo(ms);
+ return ms.ToArray();
+ }
+
+ private static IEnumerable ArchiveVersions() =>
+ Manifest.Value.Select(r => (r.Archive, r.Version)).Distinct()
+ .Select(v => new TestCaseData(v.Archive, v.Version).SetName($"{v.Archive} v{v.Version}"));
+
+ [Test, Category("ThemVsUs"), TestCaseSource(nameof(ArchiveVersions))]
+ public void ZpaqArchive_ExtractsToManifest(string archive, int version) {
+ using var r = new ZpaqReader(new MemoryStream(Load(archive)));
+ var entries = r.GetEntries(version).ToDictionary(e => e.FileName);
+ var expected = Manifest.Value.Where(m => m.Archive == archive && m.Version == version).ToArray();
+
+ Assert.Multiple(() => {
+ Assert.That(r.Errors, Is.Zero);
+ foreach (var row in expected) {
+ Assert.That(entries.ContainsKey(row.Name), $"{row.Name} is listed");
+ if (!entries.TryGetValue(row.Name, out var e))
+ continue;
+ if (row.Size < 0) {
+ Assert.That(e.IsDirectory, $"{row.Name} is a directory");
+ continue;
+ }
+ var data = r.ExtractBytes(e);
+ Assert.That(data.LongLength, Is.EqualTo(row.Size), $"{row.Name} size");
+ Assert.That(Convert.ToHexStringLower(SHA256.HashData(data)), Is.EqualTo(row.Sha256), $"{row.Name} content");
+ if (e.Size >= 0)
+ Assert.That(e.Size, Is.EqualTo(row.Size), $"{row.Name} listed size");
+ }
+ var files = entries.Values.Where(e => !e.IsDirectory).Select(e => e.FileName);
+ Assert.That(files, Is.EquivalentTo(expected.Where(m => m.Size >= 0).Select(m => m.Name)), "no extra or missing files");
+ });
+ }
+
+ [Test, Category("ThemVsUs")]
+ public void Versions_RecordUpdateAndDeletion() {
+ using var r = new ZpaqReader(new MemoryStream(Load("versions.zpaq")));
+ Assert.Multiple(() => {
+ Assert.That(r.VersionCount, Is.EqualTo(3));
+ Assert.That(r.Entries.Select(e => e.FileName), Does.Not.Contain("v/notes.txt"), "deleted in version 3");
+ Assert.That(r.GetEntries(2).Select(e => e.FileName), Does.Contain("v/notes.txt"));
+ Assert.That(r.Entries.Single(e => e.FileName == "v/a.bin").Version, Is.EqualTo(2));
+ Assert.That(r.GetVersionDate(1), Is.LessThan(r.GetVersionDate(3)!.Value));
+ });
+ }
+
+ [Test, Category("ThemVsUs")]
+ public void Dedup_SharesFragmentsBetweenIdenticalFiles([Values(0, 1, 2, 3, 4, 5)] int method) {
+ using var r = new ZpaqReader(new MemoryStream(Load($"set.m{method}.zpaq")));
+ var text = r.Entries.Single(e => e.FileName == "set/text.txt");
+ var dup = r.Entries.Single(e => e.FileName == "set/dup.txt");
+ Assert.That(dup.Fragments, Is.EqualTo(text.Fragments));
+ }
+
+ [Test, Category("ThemVsUs")]
+ public void StreamingArchive_HasNoRecordedSizes() {
+ using var r = new ZpaqReader(new MemoryStream(Load("stream.zpaq")));
+ Assert.Multiple(() => {
+ Assert.That(r.Entries.Select(e => e.FileName), Is.EquivalentTo(new[] { "st/one.txt", "st/two.bin" }));
+ Assert.That(r.Entries.Select(e => e.Size), Is.All.EqualTo(-1));
+ });
+ }
+
+ [Test, Category("ThemVsUs")]
+ public void Descriptor_ListsAndExtractsZpaqArchive() {
+ var d = new ZpaqFormatDescriptor();
+ var dir = Path.Combine(Path.GetTempPath(), $"cwb_zpaq_ref_{Guid.NewGuid():N}");
+ try {
+ d.Extract(new MemoryStream(Load("set.m3.zpaq")), dir, null, null);
+ var expected = Manifest.Value.Where(m => m.Archive == "set.m3.zpaq" && m.Size >= 0);
+ Assert.Multiple(() => {
+ foreach (var row in expected)
+ Assert.That(Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(Path.Combine(dir, row.Name)))), Is.EqualTo(row.Sha256), row.Name);
+ });
+ } finally {
+ try { Directory.Delete(dir, true); } catch { /* best effort */ }
+ }
+ }
+}
diff --git a/Compression.Tests/Zpaq/ZpaqTests.cs b/Compression.Tests/Zpaq/ZpaqTests.cs
index 212d274e4..b6f075c32 100644
--- a/Compression.Tests/Zpaq/ZpaqTests.cs
+++ b/Compression.Tests/Zpaq/ZpaqTests.cs
@@ -1,484 +1,264 @@
-using System.Text;
+using Compression.Registry;
+using Compression.Tests.Csc;
using FileFormat.Zpaq;
namespace Compression.Tests.Zpaq;
+///
+/// Archive-level behaviour of and . Whether zpaq
+/// itself agrees is pinned by (its archives) and
+/// (live, both directions).
+///
[TestFixture]
public class ZpaqTests {
- // ── Archive building helpers ─────────────────────────────────────────────
-
- ///
- /// Writes a minimal ZPAQ block header ("zPQ" + level + type) to .
- ///
- private static void WriteBlockHeader(MemoryStream ms, byte type, byte level = ZpaqConstants.Level1) {
- ms.Write(ZpaqConstants.BlockPrefix);
- ms.WriteByte(level);
- ms.WriteByte(type);
- }
+ private static readonly DateTime When = new(2026, 10, 19, 12, 0, 0, DateTimeKind.Utc);
- ///
- /// Serialises a ZPAQ level-1 journaling 'c' block payload:
- /// 8 bytes transaction FILETIME
- /// for each file: attr(1) + name(utf8 NUL) + size(8 LE)
- /// 0xFF end-of-block
- ///
- private static void WriteHeaderPayload(
- MemoryStream ms,
- long fileTime,
- IEnumerable<(string Name, long Size, byte Attr)> files) {
- ms.Write(BitConverter.GetBytes(fileTime));
- foreach (var (name, size, attr) in files) {
- ms.WriteByte(attr);
- ms.Write(Encoding.UTF8.GetBytes(name));
- ms.WriteByte(0); // null terminator
- ms.Write(BitConverter.GetBytes(size));
+ private static byte[] Write(string method, params (string Name, byte[]? Data)[] entries) {
+ using var ms = new MemoryStream();
+ using (var w = new ZpaqWriter(ms, leaveOpen: true, method) { TransactionDate = When }) {
+ foreach (var (name, data) in entries)
+ if (data == null)
+ w.AddDirectory(name);
+ else
+ w.AddFile(name, data);
}
- ms.WriteByte(0xFF); // end-of-block
+ return ms.ToArray();
}
- /// Converts a UTC to a Windows FILETIME long.
- private static long ToFileTime(DateTime utc) =>
- utc.ToFileTimeUtc();
-
- // ── Invalid input ────────────────────────────────────────────────────────
-
- [Category("Exception")]
- [Test]
- public void Constructor_NullStream_ThrowsArgumentNullException() {
- Assert.That(
- () => new ZpaqReader(null!),
- Throws.ArgumentNullException);
+ private static Dictionary ReadAll(byte[] archive) {
+ using var r = new ZpaqReader(new MemoryStream(archive));
+ return r.Entries.ToDictionary(e => e.FileName, r.ExtractBytes);
}
- [Category("Exception")]
- [Test]
- public void Constructor_WriteOnlyStream_ThrowsArgumentException() {
- using var ms = new MemoryStream([], writable: true);
- var wo = new WriteOnlyStreamWrapper(ms);
- Assert.That(
- () => new ZpaqReader(wo),
- Throws.ArgumentException);
+ // ── round trips ──────────────────────────────────────────────────────────
+
+ [Test, Category("HappyPath"), Category("RoundTrip")]
+ public void EveryPredefinedMethod_RoundTripsMixedFiles([Values("0", "1", "2", "3", "4", "5")] string method) {
+ var text = CscSamples.Text(30_000, 11);
+ var archive = Write(method,
+ ("docs/", null),
+ ("docs/text.txt", text),
+ ("bin/x86.bin", CscSamples.X86(20_000, 12)),
+ ("bin/random.bin", CscSamples.RandomBytes(6_000, 13)),
+ ("table.bin", CscSamples.Table(12_000)),
+ ("empty.dat", []));
+
+ var files = ReadAll(archive);
+
+ Assert.Multiple(() => {
+ Assert.That(files.Keys, Is.EquivalentTo(new[] { "docs/", "docs/text.txt", "bin/x86.bin", "bin/random.bin", "table.bin", "empty.dat" }));
+ Assert.That(files["docs/text.txt"], Is.EqualTo(text));
+ Assert.That(files["bin/x86.bin"], Is.EqualTo(CscSamples.X86(20_000, 12)));
+ Assert.That(files["bin/random.bin"], Is.EqualTo(CscSamples.RandomBytes(6_000, 13)));
+ Assert.That(files["table.bin"], Is.EqualTo(CscSamples.Table(12_000)));
+ Assert.That(files["empty.dat"], Is.Empty);
+ });
}
- [Category("Exception")]
- [Test]
- public void Constructor_NoZpaqBlocks_ThrowsInvalidDataException() {
- var data = "This is not a ZPAQ archive."u8.ToArray();
- using var ms = new MemoryStream(data);
- Assert.That(
- () => new ZpaqReader(ms),
- Throws.InstanceOf());
+ [Test, Category("HappyPath"), Category("RoundTrip")]
+ public void ExplicitMethodStrings_RoundTrip([Values("x4.0ci1.1.1.1.2aw2mm16tst", "x4.3ci1", "x4.7ci1", "x4.2.12.0.7.25.1c0.0.511i2", "x4.6.4.0.3.24c0.0.511", "x4.1.4.0.3.24")] string method) {
+ var data = CscSamples.Mixed();
+ var files = ReadAll(Write(method, ("mixed.bin", data)));
+ Assert.That(files["mixed.bin"], Is.EqualTo(data));
}
- // ── Empty / minimal archives ─────────────────────────────────────────────
-
- [Category("HappyPath")]
- [Test]
- public void EmptyHeaderBlock_ProducesNoEntries() {
- // A 'c' block with timestamp but no file entries.
- using var ms = new MemoryStream();
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc)), []);
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
- Assert.That(reader.Entries, Is.Empty);
+ [Test, Category("HappyPath")]
+ public void HigherMethods_CompressTextBetter() {
+ var text = CscSamples.Text(60_000, 14);
+ var sizes = new[] { "0", "1", "3", "5" }.Select(m => Write(m, ("t.txt", text)).Length).ToArray();
+ Assert.That(sizes, Is.Ordered.Descending);
}
- [Category("HappyPath")]
- [Test]
- public void SingleFile_ParsedCorrectly() {
- var ts = new DateTime(2023, 6, 15, 12, 0, 0, DateTimeKind.Utc);
- var ft = ToFileTime(ts);
-
- using var ms = new MemoryStream();
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ft, [("readme.txt", 42L, 0x20)]);
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
-
- Assert.That(reader.Entries, Has.Count.EqualTo(1));
-
- var entry = reader.Entries[0];
- Assert.That(entry.FileName, Is.EqualTo("readme.txt"));
- Assert.That(entry.Size, Is.EqualTo(42L));
- Assert.That(entry.IsDirectory, Is.False);
- Assert.That(entry.Version, Is.GreaterThan(0));
-
- // Timestamp should be within a 2-second window of the source value.
- Assert.That(entry.LastModified, Is.Not.Null);
- Assert.That(
- Math.Abs((entry.LastModified!.Value.ToUniversalTime() - ts).TotalSeconds),
- Is.LessThanOrEqualTo(2));
+ // ── deduplication ────────────────────────────────────────────────────────
+
+ [Test, Category("HappyPath")]
+ public void IdenticalFiles_AreStoredOnce() {
+ var data = CscSamples.RandomBytes(50_000, 15);
+ var one = Write("1", ("a.bin", data));
+ var two = Write("1", ("a.bin", data), ("copy/a.bin", data));
+
+ using var r = new ZpaqReader(new MemoryStream(two));
+ var a = r.Entries.Single(e => e.FileName == "a.bin");
+ var b = r.Entries.Single(e => e.FileName == "copy/a.bin");
+ Assert.Multiple(() => {
+ Assert.That(two.Length - one.Length, Is.LessThan(400), "the copy costs only its index record");
+ Assert.That(b.Fragments, Is.EqualTo(a.Fragments));
+ Assert.That(r.ExtractBytes(b), Is.EqualTo(data));
+ });
}
- [Category("HappyPath")]
- [Test]
- public void DirectoryEntry_FlaggedCorrectly() {
- using var ms = new MemoryStream();
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(DateTime.UtcNow), [("subdir/", 0L, 0x10)]);
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
-
- Assert.That(reader.Entries, Has.Count.EqualTo(1));
- Assert.That(reader.Entries[0].IsDirectory, Is.True);
- Assert.That(reader.Entries[0].Size, Is.EqualTo(0L));
+ [Test, Category("HappyPath")]
+ public void SharedContentInsideDifferentFiles_IsDeduplicatedByFragment() {
+ var common = CscSamples.RandomBytes(300_000, 16);
+ var x = (byte[])[.. CscSamples.RandomBytes(20_000, 17), .. common];
+ var y = (byte[])[.. CscSamples.RandomBytes(30_000, 18), .. common];
+ var together = Write("1", ("x", x), ("y", y)).Length;
+ var apart = Write("1", ("x", x)).Length + Write("1", ("y", y)).Length;
+ var files = ReadAll(Write("1", ("x", x), ("y", y)));
+ Assert.Multiple(() => {
+ Assert.That(together, Is.LessThan(apart - 200_000), "content-defined fragments realign after the differing prefix");
+ Assert.That(files["x"], Is.EqualTo(x));
+ Assert.That(files["y"], Is.EqualTo(y));
+ });
}
- [Category("HappyPath")]
- [Test]
- public void MultipleFilesInOneTransaction_AllParsed() {
- using var ms = new MemoryStream();
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(DateTime.UtcNow), [
- ("a.txt", 10L, 0x20),
- ("b.txt", 20L, 0x20),
- ("c.txt", 30L, 0x20),
- ]);
+ // ── boundaries ───────────────────────────────────────────────────────────
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
-
- Assert.That(reader.Entries, Has.Count.EqualTo(3));
- var names = reader.Entries.Select(e => e.FileName).ToList();
- Assert.That(names, Does.Contain("a.txt"));
- Assert.That(names, Does.Contain("b.txt"));
- Assert.That(names, Does.Contain("c.txt"));
+ [Test, Category("BoundaryCase")]
+ public void FileSizesAroundFragmentLimits_RoundTrip([Values(1, 4095, 4096, 4097, 520_191, 520_192, 520_193, 1_100_000)] int size) {
+ var data = CscSamples.RandomBytes(size, size);
+ var files = ReadAll(Write("1", ("f.bin", data)));
+ Assert.That(files["f.bin"], Is.EqualTo(data));
}
- // ── Journaling (multi-transaction) ───────────────────────────────────────
-
- [Category("HappyPath")]
- [Test]
- public void LaterTransaction_SupersedesEarlierForSameFile() {
- // Transaction 1: file.txt has size 100.
- // Transaction 2: file.txt has size 200 (update).
- // Expected: Entries contains size 200.
- using var ms = new MemoryStream();
-
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc)),
- [("file.txt", 100L, 0x20)]);
-
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(new DateTime(2021, 1, 1, 0, 0, 0, DateTimeKind.Utc)),
- [("file.txt", 200L, 0x20)]);
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
-
- Assert.That(reader.Entries, Has.Count.EqualTo(1));
- Assert.That(reader.Entries[0].Size, Is.EqualTo(200L));
+ [Test, Category("BoundaryCase")]
+ public void DataLargerThanOneBlock_SpansSeveralBlocks() {
+ // Method 10 = 1 MiB blocks, so 2.5 MiB of incompressible data needs three d blocks.
+ var data = CscSamples.RandomBytes(2_600_000, 19);
+ var archive = Write("10", ("big.bin", data));
+ var files = ReadAll(archive);
+ Assert.Multiple(() => {
+ Assert.That(files["big.bin"], Is.EqualTo(data));
+ Assert.That(CountOccurrences(archive, "d0000000"u8.ToArray()), Is.GreaterThanOrEqualTo(3));
+ });
}
- [Category("HappyPath")]
- [Test]
- public void TwoTransactions_DifferentFiles_BothPresent() {
- using var ms = new MemoryStream();
-
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(DateTime.UtcNow), [("first.bin", 5L, 0x20)]);
-
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(DateTime.UtcNow), [("second.bin", 7L, 0x20)]);
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
-
- Assert.That(reader.Entries, Has.Count.EqualTo(2));
+ [Test, Category("BoundaryCase")]
+ public void OnlyDirectories_ProduceAnIndexWithoutDataBlocks() {
+ var files = ReadAll(Write("1", ("a/", null), ("a/b/", null)));
+ Assert.That(files.Keys, Is.EquivalentTo(new[] { "a/", "a/b/" }));
}
- // ── Data block / CompressedSize ──────────────────────────────────────────
-
- [Category("HappyPath")]
- [Test]
- public void DataBlock_CompressedSizePopulated() {
- const int dataPayloadBytes = 256;
-
- using var ms = new MemoryStream();
-
- // Header block with one file.
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(DateTime.UtcNow), [("data.bin", 1000L, 0x20)]);
-
- // Data block with a known-size payload.
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeData);
- ms.Write(new byte[dataPayloadBytes]); // arbitrary payload
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
-
- Assert.That(reader.Entries, Has.Count.EqualTo(1));
- // CompressedSize should reflect the data block bytes we wrote.
- Assert.That(reader.Entries[0].CompressedSize, Is.GreaterThan(0));
+ [Test, Category("BoundaryCase")]
+ public void NoEntries_WritesNothing_AndEmptyInputReadsAsEmptyArchive() {
+ var archive = Write("1");
+ using var r = new ZpaqReader(new MemoryStream(archive));
+ Assert.Multiple(() => {
+ Assert.That(archive, Is.Empty, "zpaq truncates an update that adds nothing");
+ Assert.That(r.Entries, Is.Empty);
+ Assert.That(r.VersionCount, Is.Zero);
+ });
}
- // ── Extract ──────────────────────────────────────────────────────────────
-
- [Category("Exception")]
- [Test]
- public void Extract_WithoutStoredDataBlock_ThrowsNotSupportedException() {
- // A header that declares a size but carries no STORED data ('d') block has
- // no recoverable bytes (a ZPAQL-compressed block would need a VM), so
- // Extract reports the unsupported case rather than returning empty.
+ [Test, Category("HappyPath")]
+ public void Utf8Names_AndDates_Survive() {
+ var stamp = new DateTime(2001, 2, 3, 4, 5, 6, DateTimeKind.Utc);
using var ms = new MemoryStream();
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(DateTime.UtcNow), [("f.txt", 1L, 0x20)]);
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
-
- Assert.That(reader.Entries, Has.Count.EqualTo(1));
- Assert.That(
- () => reader.Extract(reader.Entries[0]),
- Throws.InstanceOf());
+ using (var w = new ZpaqWriter(ms, leaveOpen: true) { TransactionDate = When })
+ w.AddFile("ünï/cödé ✓.txt", "x"u8.ToArray(), stamp);
+ using var r = new ZpaqReader(new MemoryStream(ms.ToArray()));
+ var e = r.Entries.Single();
+ Assert.Multiple(() => {
+ Assert.That(e.FileName, Is.EqualTo("ünï/cödé ✓.txt"));
+ Assert.That(e.LastModified, Is.EqualTo(stamp));
+ Assert.That(r.GetVersionDate(1), Is.EqualTo(When));
+ Assert.That(e.Version, Is.EqualTo(1));
+ });
}
- [Category("HappyPath")]
- [Test]
- public void Writer_Then_Reader_RoundTripsStoredContent() {
- // ZpaqWriter stores data uncompressed in 'd' blocks; ZpaqReader must recover
- // the exact bytes for each file it lists (the conversion-matrix contract).
- var files = new (string Name, byte[] Data)[] {
- ("hello.txt", "hello conversion matrix"u8.ToArray()),
- ("data.bin", Enumerable.Range(0, 256).Select(i => (byte)i).ToArray()),
- ("big.dat", Enumerable.Range(0, 4096).Select(i => (byte)((i * 31 + 7) & 0xFF)).ToArray()),
- };
-
- using var ms = new MemoryStream();
- using (var w = new ZpaqWriter(ms, leaveOpen: true))
- foreach (var (name, data) in files)
- w.AddFile(name, data);
+ // ── stream behaviour and errors ──────────────────────────────────────────
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
-
- foreach (var (name, data) in files) {
- var entry = reader.Entries.Single(e => e.FileName == name);
- using var es = reader.Extract(entry);
- using var got = new MemoryStream();
- es.CopyTo(got);
- Assert.That(got.ToArray(), Is.EqualTo(data), $"content of '{name}' must round-trip");
- }
+ [Test, Category("HappyPath")]
+ public void NonSeekableInput_IsBuffered() {
+ var archive = Write("1", ("a.txt", "hello"u8.ToArray()));
+ using var r = new ZpaqReader(new ForwardOnlyStream(archive));
+ Assert.That(r.ExtractBytes(r.Entries.Single()), Is.EqualTo("hello"u8.ToArray()));
}
- // ── Non-seekable stream ──────────────────────────────────────────────────
+ [Test, Category("Exception")]
+ public void Constructor_NullStream_Throws() => Assert.That(() => new ZpaqReader(null!), Throws.ArgumentNullException);
- [Category("HappyPath")]
- [Test]
- public void NonSeekableStream_ParsedWithoutError() {
- using var ms = new MemoryStream();
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(DateTime.UtcNow), [("ns.txt", 99L, 0x20)]);
-
- ms.Position = 0;
- using var nsStream = new NonSeekableStreamWrapper(ms);
- using var reader = new ZpaqReader(nsStream, leaveOpen: true);
+ [Test, Category("Exception")]
+ public void NotAnArchive_ThrowsInvalidData() =>
+ Assert.That(() => new ZpaqReader(new MemoryStream("This is not a ZPAQ archive."u8.ToArray())), Throws.InstanceOf());
- Assert.That(reader.Entries, Has.Count.EqualTo(1));
- Assert.That(reader.Entries[0].FileName, Is.EqualTo("ns.txt"));
- Assert.That(reader.Entries[0].Size, Is.EqualTo(99L));
+ [Test, Category("Exception")]
+ public void EncryptedArchive_IsRefused() {
+ // An encrypted archive starts with its 32-byte random salt (never '7' or 'z' first).
+ var salt = Enumerable.Range(1, 64).Select(i => (byte)(i * 37)).ToArray();
+ Assert.That(() => new ZpaqReader(new MemoryStream(salt)), Throws.InstanceOf().With.Message.Contains("encrypted"));
}
- // ── leaveOpen ────────────────────────────────────────────────────────────
-
- [Category("HappyPath")]
- [Test]
- public void LeaveOpen_True_StreamRemainsUsable() {
- using var ms = new MemoryStream();
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(DateTime.UtcNow), [("keep.txt", 1L, 0x20)]);
- ms.Position = 0;
-
- using (var reader = new ZpaqReader(ms, leaveOpen: true)) {
- Assert.That(reader.Entries, Has.Count.EqualTo(1));
- }
-
- // Stream must still be accessible after reader disposal.
- Assert.That(ms.CanRead, Is.True);
+ [Test, Category("Exception")]
+ public void CorruptedData_FailsTheFragmentChecksum() {
+ var data = CscSamples.Text(40_000, 20);
+ var archive = Write("0", ("t.txt", data));
+ // Method 0 stores fragments verbatim, so flipping a byte of the text corrupts content, not structure.
+ var at = IndexOf(archive, data.AsSpan(1000, 32).ToArray());
+ archive[at + 5] ^= 0x20;
+ using var r = new ZpaqReader(new MemoryStream(archive));
+ Assert.That(() => r.ExtractBytes(r.Entries.Single()), Throws.InstanceOf().With.Message.Contains("checksum"));
}
- [Category("HappyPath")]
- [Test]
- public void LeaveOpen_False_StreamDisposedWithReader() {
- var ms = new MemoryStream();
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(DateTime.UtcNow), [("close.txt", 1L, 0x20)]);
- ms.Position = 0;
-
- var reader = new ZpaqReader(ms, leaveOpen: false);
- reader.Dispose();
-
- Assert.That(ms.CanRead, Is.False);
- }
-
- // ── Prefixed garbage / robustness ────────────────────────────────────────
-
- [Category("HappyPath")]
- [Test]
- public void GarbageBeforeBlock_IgnoredSuccessfully() {
- using var ms = new MemoryStream();
-
- // Write 200 bytes of noise first.
- ms.Write(new byte[200]);
-
- // Then a valid block.
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
- WriteHeaderPayload(ms, ToFileTime(DateTime.UtcNow), [("after_noise.txt", 7L, 0x20)]);
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
-
- Assert.That(reader.Entries, Has.Count.EqualTo(1));
- Assert.That(reader.Entries[0].FileName, Is.EqualTo("after_noise.txt"));
- }
-
- [Category("EdgeCase")]
- [Test]
- public void ZpaqPrefixInPayload_NotMisidentifiedAsNewBlock() {
- // The payload of a 'c' block itself contains the bytes "zPQ" — the reader
- // must not misidentify that sequence as the start of a new block while
- // parsing the payload (the scanner only looks for "zPQ" between blocks).
+ [Test, Category("Exception")]
+ public void Writer_RejectsBadArguments() {
using var ms = new MemoryStream();
-
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader);
-
- // Build a payload that embeds "zPQ" inside a filename.
- var ft = ToFileTime(DateTime.UtcNow);
- ms.Write(BitConverter.GetBytes(ft));
- ms.WriteByte(0x20); // attr
- ms.Write("file_zPQ_name.txt"u8);
- ms.WriteByte(0); // null terminator
- ms.Write(BitConverter.GetBytes(55L)); // size
- ms.WriteByte(0xFF); // end-of-block
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
-
- // The entry must be present and its filename must be intact.
- Assert.That(reader.Entries, Has.Count.EqualTo(1));
- Assert.That(reader.Entries[0].FileName, Does.Contain("zPQ"));
+ Assert.Multiple(() => {
+ Assert.That(() => new ZpaqWriter(ms, true, "q"), Throws.ArgumentException);
+ Assert.That(() => new ZpaqWriter(ms, true, ""), Throws.ArgumentException);
+ using var w = new ZpaqWriter(ms, true);
+ Assert.That(() => w.AddFile("dir/", []), Throws.ArgumentException);
+ Assert.That(() => w.AddFile(null!, []), Throws.ArgumentNullException);
+ });
}
- // ── Writer round-trip tests ──────────────────────────────────────────────
+ // ── descriptor ───────────────────────────────────────────────────────────
- [Category("HappyPath")]
- [Category("RoundTrip")]
- [Test]
- public void Writer_RoundTrip_SingleFile() {
+ [Test, Category("HappyPath")]
+ public void Descriptor_CreateHonoursLevel_AndListsEntries() {
+ var d = new ZpaqFormatDescriptor();
+ var text = CscSamples.Text(20_000, 21);
using var ms = new MemoryStream();
- using (var w = new ZpaqWriter(ms, leaveOpen: true))
- w.AddFile("hello.txt", "Hello, ZPAQ!"u8.ToArray());
-
+ d.Create(ms, [ArchiveInputInfo.InMemory("a/t.txt", text)], new FormatCreateOptions { Level = 4 });
ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
- Assert.That(reader.Entries, Has.Count.EqualTo(1));
- Assert.That(reader.Entries[0].FileName, Is.EqualTo("hello.txt"));
- Assert.That(reader.Entries[0].Size, Is.EqualTo(12));
+ var list = d.List(ms, null);
+ Assert.Multiple(() => {
+ Assert.That(list.Select(e => e.Name), Is.EqualTo(new[] { "a/t.txt" }));
+ Assert.That(list[0].OriginalSize, Is.EqualTo(text.Length));
+ });
}
- [Category("HappyPath")]
- [Category("RoundTrip")]
- [Test]
- public void Writer_RoundTrip_MultipleFiles() {
- using var ms = new MemoryStream();
- using (var w = new ZpaqWriter(ms, leaveOpen: true)) {
- w.AddFile("a.txt", new byte[100]);
- w.AddFile("b.txt", new byte[200]);
- }
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
- Assert.That(reader.Entries, Has.Count.EqualTo(2));
- var names = reader.Entries.Select(e => e.FileName).ToList();
- Assert.That(names, Does.Contain("a.txt"));
- Assert.That(names, Does.Contain("b.txt"));
+ [Test, Category("Exception")]
+ public void Descriptor_List_FallsBackInsteadOfThrowing() {
+ var d = new ZpaqFormatDescriptor();
+ var list = d.List(new MemoryStream("7kSt garbage that is not a block"u8.ToArray()), null);
+ Assert.That(list.Select(e => e.Name), Is.EqualTo(new[] { "FULL.zpaq", "metadata.ini" }));
}
- [Category("HappyPath")]
- [Category("RoundTrip")]
- [Test]
- public void Writer_RoundTrip_Directory() {
- using var ms = new MemoryStream();
- using (var w = new ZpaqWriter(ms, leaveOpen: true)) {
- w.AddDirectory("mydir/");
- w.AddFile("mydir/file.txt", [1, 2, 3]);
- }
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
- var dirs = reader.Entries.Where(e => e.IsDirectory).ToList();
- var files = reader.Entries.Where(e => !e.IsDirectory).ToList();
- Assert.That(dirs, Has.Count.EqualTo(1));
- Assert.That(files, Has.Count.EqualTo(1));
+ [Test, Category("HappyPath")]
+ public void Descriptor_MethodMapping() {
+ Assert.Multiple(() => {
+ Assert.That(ZpaqFormatDescriptor.MethodFor(new FormatCreateOptions()), Is.EqualTo("1"));
+ Assert.That(ZpaqFormatDescriptor.MethodFor(new FormatCreateOptions { Level = 9 }), Is.EqualTo("5"));
+ Assert.That(ZpaqFormatDescriptor.MethodFor(new FormatCreateOptions("x4.3ci1")), Is.EqualTo("x4.3ci1"));
+ Assert.That(ZpaqFormatDescriptor.MethodFor(new FormatCreateOptions("zpaq")), Is.EqualTo("1"));
+ });
}
- [Category("HappyPath")]
- [Test]
- public void Writer_Magic_IsZPQ() {
- using var ms = new MemoryStream();
- using (var w = new ZpaqWriter(ms, leaveOpen: true))
- w.AddFile("x", [1]);
+ private static int IndexOf(byte[] haystack, byte[] needle) => haystack.AsSpan().IndexOf(needle);
- ms.Position = 0;
- Span magic = stackalloc byte[3];
- ms.ReadExactly(magic);
- Assert.That(magic[0], Is.EqualTo((byte)'z'));
- Assert.That(magic[1], Is.EqualTo((byte)'P'));
- Assert.That(magic[2], Is.EqualTo((byte)'Q'));
- }
-
- // ── Level 2 blocks accepted ───────────────────────────────────────────────
-
- [Category("HappyPath")]
- [Test]
- public void Level2Block_ParsedWithoutError() {
- using var ms = new MemoryStream();
- WriteBlockHeader(ms, ZpaqConstants.BlockTypeHeader, ZpaqConstants.Level2);
- WriteHeaderPayload(ms, ToFileTime(DateTime.UtcNow), [("v2.txt", 3L, 0x20)]);
-
- ms.Position = 0;
- using var reader = new ZpaqReader(ms, leaveOpen: true);
-
- Assert.That(reader.Entries, Has.Count.EqualTo(1));
- Assert.That(reader.Entries[0].FileName, Is.EqualTo("v2.txt"));
+ private static int CountOccurrences(byte[] haystack, byte[] needle) {
+ var count = 0;
+ for (var span = haystack.AsSpan(); ;) {
+ var i = span.IndexOf(needle);
+ if (i < 0)
+ return count;
+ ++count;
+ span = span[(i + needle.Length)..];
+ }
}
-}
-// ── Test stream wrappers ────────────────────────────────────────────────────
-
-/// Wraps a stream and exposes it as non-seekable.
-internal sealed class NonSeekableStreamWrapper(Stream inner) : Stream {
- public override bool CanRead => inner.CanRead;
- public override bool CanSeek => false;
- public override bool CanWrite => inner.CanWrite;
- public override long Length => throw new NotSupportedException();
- public override long Position {
- get => throw new NotSupportedException();
- set => throw new NotSupportedException();
+ private sealed class ForwardOnlyStream(byte[] data) : Stream {
+ private readonly MemoryStream _inner = new(data);
+ public override bool CanRead => true;
+ public override bool CanSeek => false;
+ public override bool CanWrite => false;
+ public override long Length => throw new NotSupportedException();
+ public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
+ public override void Flush() { }
+ public override int Read(byte[] buffer, int offset, int count) => this._inner.Read(buffer, offset, count);
+ public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
+ public override void SetLength(long value) => throw new NotSupportedException();
+ public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
}
- public override void Flush() => inner.Flush();
- public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count);
- public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
- public override void SetLength(long value) => throw new NotSupportedException();
- public override void Write(byte[] buffer, int offset, int count) => inner.Write(buffer, offset, count);
- protected override void Dispose(bool disposing) { if (disposing) inner.Dispose(); base.Dispose(disposing); }
-}
-
-/// Wraps a stream and exposes it as write-only (CanRead = false).
-internal sealed class WriteOnlyStreamWrapper(Stream inner) : Stream {
- public override bool CanRead => false;
- public override bool CanSeek => false;
- public override bool CanWrite => true;
- public override long Length => throw new NotSupportedException();
- public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
- public override void Flush() => inner.Flush();
- public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
- public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
- public override void SetLength(long value) => throw new NotSupportedException();
- public override void Write(byte[] buffer, int offset, int count) => inner.Write(buffer, offset, count);
- protected override void Dispose(bool disposing) { if (disposing) inner.Dispose(); base.Dispose(disposing); }
}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/THIRD-PARTY-NOTICE.libzpaq.txt b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/THIRD-PARTY-NOTICE.libzpaq.txt
new file mode 100644
index 000000000..4e93c232e
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/THIRD-PARTY-NOTICE.libzpaq.txt
@@ -0,0 +1,67 @@
+libzpaq / zpaq 7.15 — converted to C#
+
+Source: http://mattmahoney.net/dc/zpaq715.zip (zpaq v7.15, Aug 17 2016)
+Author: Matt Mahoney
+Licence: public domain (The Unlicense, reproduced below) for zpaq.cpp,
+ libzpaq.h and libzpaq.cpp. The libdivsufsort-lite section of
+ libzpaq.cpp (MIT) was NOT taken: suffix arrays are built by
+ ZpaqSuffixArray.cs, an SA-IS implementation written from the
+ Nong/Zhang/Chan paper. A suffix array is unique, so the BWT and
+ LZ77 output is unchanged.
+Spec: The ZPAQ Open Standard Format for Highly Compressed Data, Level 2,
+ http://mattmahoney.net/dc/zpaq206.pdf
+
+The files in this directory are a conversion into this codebase, not a
+verbatim copy:
+
+ libzpaq.cpp ZPAQL (interpreter) -> ZpaqlMachine.cs
+ libzpaq.cpp Predictor (interpreter) -> ZpaqPredictor.cs
+ libzpaq.cpp Decoder, Encoder,
+ PostProcessor -> ZpaqCoder.cs
+ libzpaq.cpp Decompresser -> ZpaqDecompresser.cs
+ libzpaq.cpp Compiler -> ZpaqCompiler.cs
+ libzpaq.cpp Compressor -> ZpaqCompressor.cs
+ libzpaq.cpp makeConfig, compressBlock,
+ e8e9 -> ZpaqMethods.cs
+ libzpaq.cpp LZBuffer -> ZpaqLzBuffer.cs
+ libzpaq.cpp StringBuffer, SHA1 use -> ZpaqIo.cs
+ zpaq.cpp Jidac::read_archive,
+ extract -> ZpaqReader.cs
+ zpaq.cpp Jidac::add -> ZpaqWriter.cs
+
+Constant tables (bit-history state table, CM/MATCH divisor tables, squash
+and stretch tables, the add() analysis divisor table, the locator tag, the
+ZPAQL opcode list and the generated ZPAQL programs for methods 0..5) are
+copied exactly, as the format requires.
+
+Not converted: the x86/x64 JIT, AES-256 encryption and scrypt key stretching
+(encrypted archives are refused), multi-part and index (-index) archives,
+and appending a new version to an existing archive (the writer emits a
+single-version archive; modify rebuilds it).
+
+---------------------------------------------------------------------
+
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqBuildingBlock.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqBuildingBlock.cs
index e92e1d9a0..b297e0fcb 100644
--- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqBuildingBlock.cs
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqBuildingBlock.cs
@@ -1,238 +1,63 @@
-using System.Buffers.Binary;
using Compression.Registry;
namespace FileFormat.Zpaq;
///
-/// Exposes the ZPAQ context-mixing codec — the compression stage that the ZPAQ
-/// journaling archiver wraps — as a benchmarkable building block.
-/// Prepends a 4-byte LE uncompressed size header for round-trip support.
+/// ZPAQ compression as a building block: the input coded as a plain ZPAQ stream (locator tag,
+/// blocks of one segment, SHA-1 trailers) with zpaq's method 4 — an order-0 ICM, an ISSE chain
+/// up to order 6, a match model and a mixer, all described in the block's own ZPAQL header.
///
///
-///
-/// ZPAQ (Matt Mahoney, "The ZPAQ Open Standard Format for Highly Compressed Data")
-/// codes one bit at a time: an array of prediction components supplies a probability
-/// for the next bit, a binary arithmetic coder codes the bit against that probability,
-/// and a ZPAQL bytecode program (HCOMP) recomputes the context hashes in H[] after
-/// every whole byte. runs the program and the components,
-/// and do the coding,
-/// and this block supplies one concrete configuration and drives the loop.
-///
-///
-/// The configuration is four direct context models over hashed orders 1 to 4, built
-/// by the HCOMP program in . Per the format, the component context
-/// is refined with the bits of the partly coded byte before each bit is coded; that
-/// refinement belongs to the predictor rather than to HCOMP, so it is applied here.
-///
-///
-/// The wire format is a 4-byte little-endian uncompressed length followed by the
-/// coded bytes. An empty message is the header alone. The decoder learns how many
-/// bytes to produce from the header, so the coded stream carries no end marker.
-///
-///
-/// This is one configuration among the many the format allows, and it combines the
-/// component predictions the way does, by averaging
-/// rather than through a mixer chain. It is therefore a ZPAQ-model codec, not a
-/// bit-exact producer of any particular reference archive's streams.
-///
-///
-/// This block is the codec only. It is deliberately not the archive container:
-/// and model the journaling archive
-/// (transactions, filenames, timestamps, SHA-1 index) and store their payloads
-/// verbatim, so they carry no compression stage of their own to expose. Deduplication
-/// and versioning, which the real container layers on top of those, are likewise not
-/// implemented.
-///
+/// The output is what libzpaq's compress(in, out, "4") produces and any ZPAQ Level 2
+/// decoder (zpaq x, unzpaq) reads; accepts any ZPAQ stream,
+/// whatever its method. Blocks hold at most 16 MiB - 4 KiB each, as in libzpaq.
///
public sealed class ZpaqBuildingBlock : IBuildingBlock {
-
- /// Number of context models, one per hashed order 1..4.
- private const int ComponentCount = 4;
-
- /// Log2 of the counter table size of each context model.
- private const int ContextModelBits = 16;
-
- /// Counter adaptation rate of each context model.
- private const int ContextModelRate = 4;
-
- /// Log2 of the H[] context array size. Must cover .
- private const int HashArrayBits = 3;
-
- /// Log2 of the M[] history buffer size used by the HCOMP program.
- private const int HistoryBufferBits = 16;
-
- /// Size of the uncompressed-length header, in bytes.
- private const int SizeHeaderBytes = 4;
-
- /// Odd multiplier that spreads the partly coded byte across the low context bits.
- private const uint PartialByteMultiplier = 0x9E3779B1;
-
- ///
- /// HCOMP program: hashes orders 1 to 4 of the byte history into H[0..3].
- ///
- ///
- /// On entry the ZPAQL A register holds the byte just coded, and R[0] is the write
- /// cursor into the M[] history buffer. Each order reaches one byte further back and
- /// folds it into the previous order's hash with the HASH instruction
- /// (A = (A + M[B] + 512) * 773), so order n hashes the last n bytes.
- ///
- /// R1=A A=R0 B=A A=R1 *B=A A=R0 A++ R0=A ; M[pos++] = byte
- /// A=R0 A-=1 B=A A=0 HASH H0=A ; order 1
- /// A=R0 A-=2 B=A A=H0 HASH H1=A ; order 2
- /// A=R0 A-=3 B=A A=H1 HASH H2=A ; order 3
- /// A=R0 A-=4 B=A A=H2 HASH H3=A ; order 4
- /// HALT
- ///
- ///
- private static ReadOnlySpan Hcomp => [
- // Append the byte to the history buffer and advance the cursor.
- 48, 1, // R[1] = A
- 49, 0, // A = R[0]
- 6, // B = A
- 49, 1, // A = R[1]
- 9, // M[B] = A
- 49, 0, // A = R[0]
- 2, // A++
- 48, 0, // R[0] = A
-
- // Order 1: hash of M[pos-1] alone.
- 49, 0, 3, 6, // A = R[0]; A -= 1; B = A
- 5, // A = 0
- 225, // HASH
- 82, 0, // H[0] = A
-
- // Order 2: the order-1 hash folded with M[pos-2].
- 49, 0, 3, 3, 6, // A = R[0]; A -= 2; B = A
- 83, 0, // A = H[0]
- 225, // HASH
- 82, 1, // H[1] = A
-
- // Order 3: the order-2 hash folded with M[pos-3].
- 49, 0, 3, 3, 3, 6,
- 83, 1, // A = H[1]
- 225, // HASH
- 82, 2, // H[2] = A
-
- // Order 4: the order-3 hash folded with M[pos-4].
- 49, 0, 3, 3, 3, 3, 6,
- 83, 2, // A = H[2]
- 225, // HASH
- 82, 3, // H[3] = A
-
- 0, // HALT
- ];
+ private const string Method = "4";
+ private const int BlockSize = (1 << 24) - 4096;
///
public string Id => "BB_Zpaq";
///
public string DisplayName => "ZPAQ";
///
- public string Description => "ZPAQ context-mixing model over hashed orders 1-4, contexts built by the ZPAQL virtual machine, coded bit by bit by a carry-propagating binary range coder";
+ public string Description => "ZPAQ Level 2 stream, method 4: ICM + ISSE chain + match model + mixer, contexts computed by the block's ZPAQL program";
///
public AlgorithmFamily Family => AlgorithmFamily.ContextMixing;
///
public byte[] Compress(ReadOnlySpan data) {
- var output = new MemoryStream();
- Span header = stackalloc byte[SizeHeaderBytes];
- BinaryPrimitives.WriteInt32LittleEndian(header, data.Length);
- output.Write(header);
-
- if (data.Length == 0)
- return output.ToArray();
-
- var vm = CreateVm();
- var contexts = new uint[ComponentCount];
- var coder = new ZpaqRangeEncoder(output);
-
- foreach (var value in data) {
- var partial = 1;
- for (var shift = 7; shift >= 0; --shift) {
- var bit = (value >> shift) & 1;
- ApplyContexts(vm, contexts, partial);
- coder.EncodeBit(bit, vm.Predict());
- vm.Update(bit);
- partial = (partial << 1) | bit;
- }
-
- vm.RunHcomp(value);
- CaptureContexts(vm, contexts);
+ var output = new ZpaqBuffer();
+ for (var offset = 0; offset < data.Length; offset += BlockSize) {
+ var chunk = new ZpaqBuffer(data.Slice(offset, Math.Min(BlockSize, data.Length - offset)));
+ ZpaqMethods.CompressBlock(chunk, output, Method, storeSha1: true);
}
-
- coder.Flush();
return output.ToArray();
}
///
- public byte[] Decompress(ReadOnlySpan data) {
- var originalSize = BinaryPrimitives.ReadInt32LittleEndian(data);
- if (originalSize == 0)
- return [];
-
- var payload = data[SizeHeaderBytes..].ToArray();
- var coder = new ZpaqRangeDecoder(payload, 0);
-
- var vm = CreateVm();
- var contexts = new uint[ComponentCount];
- var result = new byte[originalSize];
+ public byte[] Decompress(ReadOnlySpan data) => ZpaqStream.Decompress(data);
+}
- for (var index = 0; index < result.Length; ++index) {
- var partial = 1;
- for (var shift = 7; shift >= 0; --shift) {
- ApplyContexts(vm, contexts, partial);
- var bit = coder.DecodeBit(vm.Predict());
- vm.Update(bit);
- partial = (partial << 1) | bit;
+/// Decodes a whole ZPAQ stream (all blocks and segments, filenames and comments ignored).
+internal static class ZpaqStream {
+ /// Concatenates the decoded data of every segment, verifying stored SHA-1 trailers.
+ public static byte[] Decompress(ReadOnlySpan data) {
+ var input = new ZpaqBuffer(data);
+ var output = new ZpaqBuffer();
+ var d = new ZpaqDecompresser(input);
+ d.SetOutput(output);
+ while (d.FindBlock()) {
+ while (d.FindFilename() != null) {
+ d.ReadComment();
+ var sha1 = new ZpaqSha1();
+ d.SetSha1(sha1);
+ d.Decompress();
+ var stored = d.ReadSegmentEnd();
+ if (stored != null && !stored.AsSpan().SequenceEqual(sha1.Result()))
+ ZpaqError.Throw("segment checksum failed");
}
-
- var value = (byte)partial;
- result[index] = value;
- vm.RunHcomp(value);
- CaptureContexts(vm, contexts);
}
-
- return result;
- }
-
- ///
- /// Builds the VM for one message: four context models fed by .
- ///
- private static ZpaqlVm CreateVm() {
- var components = new Component[ComponentCount];
- for (var i = 0; i < components.Length; ++i)
- components[i] = new Component {
- Type = ComponentType.Cm,
- Param1 = ContextModelBits,
- Param2 = ContextModelRate,
- };
-
- return new ZpaqlVm(Hcomp.ToArray(), [], HashArrayBits, HistoryBufferBits, components);
- }
-
- ///
- /// Records the per-byte context hashes the HCOMP program has just written, so
- /// they can be re-derived for each bit of the next byte.
- ///
- private static void CaptureContexts(ZpaqlVm vm, uint[] contexts) {
- for (var i = 0; i < contexts.Length; ++i)
- contexts[i] = vm.H[i];
- }
-
- ///
- /// Refines each component's context with the bits of the partly coded byte.
- ///
- ///
- /// The multiplier is odd, so within one model no two prefixes of the same byte
- /// can land on the same counter. The component then indexes its table by the low
- /// bits of the offset hash; those bits are the same
- /// whether or not the sum is first reduced modulo 2^32, because the table size
- /// divides 2^32.
- ///
- /// The VM whose H[] array supplies the component contexts.
- /// The per-byte hashes captured by .
- /// The bits coded so far in the current byte, preceded by a leading 1.
- private static void ApplyContexts(ZpaqlVm vm, uint[] contexts, int partial) {
- for (var i = 0; i < contexts.Length; ++i)
- vm.H[i] = contexts[i] + (uint)partial * PartialByteMultiplier;
+ return output.ToArray();
}
}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqCoder.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqCoder.cs
new file mode 100644
index 000000000..2244a634d
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqCoder.cs
@@ -0,0 +1,271 @@
+namespace FileFormat.Zpaq;
+
+/// Arithmetic decoder of a ZPAQ segment (or the stored-block reader for unmodeled blocks).
+/// Converted from libzpaq 7.15 (Matt Mahoney, public domain), class Decoder.
+internal sealed class ZpaqDecoder : IZpaqReader {
+ private const int BufferSize = 1 << 16;
+ private readonly byte[] _buf = new byte[BufferSize];
+ private readonly ZpaqPredictor _pr;
+ private uint _low = 1, _high = 0xFFFFFFFF, _curr;
+ private int _rpos, _wpos;
+
+ /// Compressed input.
+ public IZpaqReader? In;
+
+ /// Creates a decoder for the block held by .
+ public ZpaqDecoder(ZpaqlMachine z) => this._pr = new(z);
+
+ /// Bytes read ahead from but not consumed yet.
+ public int Buffered => this._wpos - this._rpos;
+
+ /// Discards the read-ahead (after the caller repositioned ).
+ public void DiscardBuffer() => this._rpos = this._wpos = 0;
+
+ ///
+ public int Get() {
+ if (this._rpos == this._wpos) {
+ this._rpos = 0;
+ this._wpos = this.In?.Read(this._buf) ?? 0;
+ }
+ return this._rpos < this._wpos ? this._buf[this._rpos++] : -1;
+ }
+
+ /// Initialises the model at the start of a block.
+ public void Init() {
+ this._pr.Init();
+ if (this._pr.IsModeled) {
+ this._low = 1;
+ this._high = 0xFFFFFFFF;
+ this._curr = 0;
+ } else
+ this._low = this._high = this._curr = 0;
+ }
+
+ /// Returns the next decoded byte, or -1 at the end of the segment.
+ public int Decompress() {
+ if (this._pr.IsModeled) {
+ if (this._curr == 0)
+ for (var i = 0; i < 4; ++i)
+ this._curr = this._curr << 8 | (uint)this.Get();
+ if (this.Decode(0) != 0) {
+ if (this._curr != 0)
+ ZpaqError.Throw("decoding end of stream");
+ return -1;
+ }
+ var c = 1;
+ while (c < 256) {
+ var p = this._pr.Predict() * 2 + 1;
+ c += c + this.Decode(p);
+ this._pr.Update(c & 1);
+ }
+ return c - 256;
+ }
+
+ if (this._curr == 0) {
+ for (var i = 0; i < 4; ++i)
+ this._curr = this._curr << 8 | (uint)this.Get();
+ if (this._curr == 0)
+ return -1;
+ }
+ --this._curr;
+ return this.Get();
+ }
+
+ /// Skips to the end of the compressed segment and returns the byte after it.
+ public int Skip() {
+ var c = -1;
+ if (this._pr.IsModeled) {
+ while (this._curr == 0)
+ this._curr = (uint)this.Get();
+ while (this._curr != 0 && (c = this.Get()) >= 0)
+ this._curr = this._curr << 8 | (uint)c;
+ while ((c = this.Get()) == 0) { }
+ return c;
+ }
+ if (this._curr == 0)
+ for (var i = 0; i < 4 && (c = this.Get()) >= 0; ++i)
+ this._curr = this._curr << 8 | (uint)c;
+ while (this._curr > 0) {
+ while (this._curr > 0) {
+ --this._curr;
+ if (this.Get() < 0)
+ ZpaqError.Throw("skipped to EOF");
+ }
+ for (var i = 0; i < 4 && (c = this.Get()) >= 0; ++i)
+ this._curr = this._curr << 8 | (uint)c;
+ }
+ if (c >= 0)
+ c = this.Get();
+ return c;
+ }
+
+ private int Decode(int p) {
+ if (this._curr < this._low || this._curr > this._high)
+ ZpaqError.Throw("archive corrupted");
+ var mid = this._low + (uint)(((ulong)(this._high - this._low) * (uint)p) >> 16);
+ int y;
+ if (this._curr <= mid) {
+ y = 1;
+ this._high = mid;
+ } else {
+ y = 0;
+ this._low = mid + 1;
+ }
+ while ((this._high ^ this._low) < 0x1000000) {
+ this._high = this._high << 8 | 255;
+ this._low <<= 8;
+ if (this._low == 0)
+ this._low = 1;
+ var c = this.Get();
+ if (c < 0)
+ ZpaqError.Throw("unexpected end of file");
+ this._curr = this._curr << 8 | (uint)c;
+ }
+ return y;
+ }
+}
+
+/// Arithmetic encoder of a ZPAQ segment (or the stored-block writer for unmodeled blocks).
+/// Converted from libzpaq 7.15 (Matt Mahoney, public domain), class Encoder.
+internal sealed class ZpaqEncoder {
+ private readonly ZpaqPredictor _pr;
+ private uint _low = 1, _high = 0xFFFFFFFF;
+ private byte[] _buf = [];
+
+ /// Compressed output.
+ public IZpaqWriter Out = ZpaqNullWriter.Instance;
+
+ /// Creates an encoder for the block held by .
+ public ZpaqEncoder(ZpaqlMachine z) => this._pr = new(z);
+
+ /// Initialises the model at the start of a block.
+ public void Init() {
+ this._low = 1;
+ this._high = 0xFFFFFFFF;
+ this._pr.Init();
+ if (!this._pr.IsModeled) {
+ this._low = 0;
+ this._buf = new byte[1 << 16];
+ }
+ }
+
+ /// Encodes byte (0..255) or the end of segment (-1).
+ public void Compress(int c) {
+ if (this._pr.IsModeled) {
+ if (c == -1)
+ this.Encode(1, 0);
+ else {
+ this.Encode(0, 0);
+ for (var i = 7; i >= 0; --i) {
+ var p = this._pr.Predict() * 2 + 1;
+ var y = c >> i & 1;
+ this.Encode(y, p);
+ this._pr.Update(y);
+ }
+ }
+ return;
+ }
+ if (this._low != 0 && (c < 0 || this._low == this._buf.Length)) {
+ this.Out.Put((int)(this._low >> 24) & 255);
+ this.Out.Put((int)(this._low >> 16) & 255);
+ this.Out.Put((int)(this._low >> 8) & 255);
+ this.Out.Put((int)this._low & 255);
+ this.Out.Write(this._buf.AsSpan(0, (int)this._low));
+ this._low = 0;
+ }
+ if (c >= 0)
+ this._buf[this._low++] = (byte)c;
+ }
+
+ private void Encode(int y, int p) {
+ var mid = this._low + (uint)(((ulong)(this._high - this._low) * (uint)p) >> 16);
+ if (y != 0)
+ this._high = mid;
+ else
+ this._low = mid + 1;
+ while ((this._high ^ this._low) < 0x1000000) {
+ this.Out.Put((int)(this._high >> 24));
+ this._high = this._high << 8 | 255;
+ this._low <<= 8;
+ if (this._low == 0)
+ this._low = 1;
+ }
+ }
+}
+
+/// Runs the PCOMP post-processor (if any) over the decoded bytes of a block.
+/// Converted from libzpaq 7.15 (Matt Mahoney, public domain), class PostProcessor.
+internal sealed class ZpaqPostProcessor {
+ private int _state;
+ private int _hsize;
+ private int _ph, _pm;
+
+ /// The PCOMP machine.
+ public readonly ZpaqlMachine Z = new();
+
+ /// Current parse state: 0 = initial, 1 = pass-through, 2..4 = loading PCOMP, 5 = running PCOMP.
+ public int State => this._state;
+
+ /// Resets for a new block with PCOMP array sizes and .
+ public void Init(int h, int m) {
+ this._state = this._hsize = 0;
+ this._ph = h;
+ this._pm = m;
+ this.Z.Clear();
+ }
+
+ /// Feeds one decoded byte (or -1 at end of segment) and returns the new state.
+ public int Write(int c) {
+ switch (this._state) {
+ case 0:
+ if (c < 0)
+ ZpaqError.Throw("unexpected EOS");
+ this._state = c + 1;
+ if (this._state > 2)
+ ZpaqError.Throw("unknown post processing type");
+ if (this._state == 1)
+ this.Z.Clear();
+ break;
+ case 1:
+ this.Z.OutC(c);
+ break;
+ case 2:
+ if (c < 0)
+ ZpaqError.Throw("unexpected EOS");
+ this._hsize = c;
+ this._state = 3;
+ break;
+ case 3:
+ if (c < 0)
+ ZpaqError.Throw("unexpected EOS");
+ this._hsize += c * 256;
+ if (this._hsize < 1)
+ ZpaqError.Throw("empty PCOMP");
+ this.Z.Header = new byte[this._hsize + 300];
+ this.Z.Cend = 8;
+ this.Z.Hbegin = this.Z.Hend = this.Z.Cend + 128;
+ this.Z.Header[4] = (byte)this._ph;
+ this.Z.Header[5] = (byte)this._pm;
+ this._state = 4;
+ break;
+ case 4:
+ if (c < 0)
+ ZpaqError.Throw("unexpected EOS");
+ this.Z.Header[this.Z.Hend++] = (byte)c;
+ if (this.Z.Hend - this.Z.Hbegin == this._hsize) {
+ this._hsize = this.Z.Cend - 2 + this.Z.Hend - this.Z.Hbegin;
+ this.Z.Header[0] = (byte)this._hsize;
+ this.Z.Header[1] = (byte)(this._hsize >> 8);
+ this.Z.InitPcomp();
+ this._state = 5;
+ }
+ break;
+ case 5:
+ this.Z.Run((uint)c);
+ if (c < 0)
+ this.Z.Flush();
+ break;
+ }
+ return this._state;
+ }
+}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqCompiler.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqCompiler.cs
new file mode 100644
index 000000000..ec174ebf2
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqCompiler.cs
@@ -0,0 +1,331 @@
+namespace FileFormat.Zpaq;
+
+///
+/// Compiles a ZPAQL configuration (comp ... hcomp ... [pcomp ... ;] ... end) into the
+/// COMP/HCOMP header of one machine and the PCOMP code of another.
+///
+/// Converted from libzpaq 7.15 (Matt Mahoney, public domain), class Compiler.
+internal sealed class ZpaqCompiler {
+ private const int Jt = 39, Jf = 47, Jmp = 63, Lj = 255;
+ private const int Post = 256, Pcomp = 257, End = 258, If = 259, IfNot = 260, Else = 261, EndIf = 262, Do = 263,
+ While = 264, Until = 265, Forever = 266, IfL = 267, IfNotL = 268, ElseL = 269;
+
+ private static readonly string?[] ComponentNames = ["", "const", "cm", "icm", "match", "avg", "mix2", "mix", "isse", "sse", null];
+
+ private static readonly string?[] OpcodeList = [
+ "error", "a++", "a--", "a!", "a=0", "", "", "a=r",
+ "b<>a", "b++", "b--", "b!", "b=0", "", "", "b=r",
+ "c<>a", "c++", "c--", "c!", "c=0", "", "", "c=r",
+ "d<>a", "d++", "d--", "d!", "d=0", "", "", "d=r",
+ "*b<>a", "*b++", "*b--", "*b!", "*b=0", "", "", "jt",
+ "*c<>a", "*c++", "*c--", "*c!", "*c=0", "", "", "jf",
+ "*d<>a", "*d++", "*d--", "*d!", "*d=0", "", "", "r=a",
+ "halt", "out", "", "hash", "hashd", "", "", "jmp",
+ "a=a", "a=b", "a=c", "a=d", "a=*b", "a=*c", "a=*d", "a=",
+ "b=a", "b=b", "b=c", "b=d", "b=*b", "b=*c", "b=*d", "b=",
+ "c=a", "c=b", "c=c", "c=d", "c=*b", "c=*c", "c=*d", "c=",
+ "d=a", "d=b", "d=c", "d=d", "d=*b", "d=*c", "d=*d", "d=",
+ "*b=a", "*b=b", "*b=c", "*b=d", "*b=*b", "*b=*c", "*b=*d", "*b=",
+ "*c=a", "*c=b", "*c=c", "*c=d", "*c=*b", "*c=*c", "*c=*d", "*c=",
+ "*d=a", "*d=b", "*d=c", "*d=d", "*d=*b", "*d=*c", "*d=*d", "*d=",
+ "", "", "", "", "", "", "", "",
+ "a+=a", "a+=b", "a+=c", "a+=d", "a+=*b", "a+=*c", "a+=*d", "a+=",
+ "a-=a", "a-=b", "a-=c", "a-=d", "a-=*b", "a-=*c", "a-=*d", "a-=",
+ "a*=a", "a*=b", "a*=c", "a*=d", "a*=*b", "a*=*c", "a*=*d", "a*=",
+ "a/=a", "a/=b", "a/=c", "a/=d", "a/=*b", "a/=*c", "a/=*d", "a/=",
+ "a%=a", "a%=b", "a%=c", "a%=d", "a%=*b", "a%=*c", "a%=*d", "a%=",
+ "a&=a", "a&=b", "a&=c", "a&=d", "a&=*b", "a&=*c", "a&=*d", "a&=",
+ "a&~a", "a&~b", "a&~c", "a&~d", "a&~*b", "a&~*c", "a&~*d", "a&~",
+ "a|=a", "a|=b", "a|=c", "a|=d", "a|=*b", "a|=*c", "a|=*d", "a|=",
+ "a^=a", "a^=b", "a^=c", "a^=d", "a^=*b", "a^=*c", "a^=*d", "a^=",
+ "a<<=a", "a<<=b", "a<<=c", "a<<=d", "a<<=*b", "a<<=*c", "a<<=*d", "a<<=",
+ "a>>=a", "a>>=b", "a>>=c", "a>>=d", "a>>=*b", "a>>=*c", "a>>=*d", "a>>=",
+ "a==a", "a==b", "a==c", "a==d", "a==*b", "a==*c", "a==*d", "a==",
+ "aa", "a>b", "a>c", "a>d", "a>*b", "a>*c", "a>*d", "a>",
+ "", "", "", "", "", "", "", "",
+ "", "", "", "", "", "", "", "lj",
+ "post", "pcomp", "end", "if", "ifnot", "else", "endif", "do",
+ "while", "until", "forever", "ifl", "ifnotl", "elsel", ";", null,
+ ];
+
+ private readonly string _in;
+ private int _pos;
+ private readonly int[] _args;
+ private int _line = 1;
+ private int _state;
+ private readonly Stack _ifStack = new();
+ private readonly Stack _doStack = new();
+
+ /// The text between pcomp and ;: the preprocessor command the configuration names.
+ public string PcompCommand { get; private set; } = "";
+
+ ///
+ /// Compiles , substituting $1..$9 from ,
+ /// into (COMP + HCOMP) and (PCOMP).
+ ///
+ public ZpaqCompiler(string config, int[]? args, ZpaqlMachine hz, ZpaqlMachine pz) {
+ this._in = config;
+ this._args = args ?? new int[9];
+ hz.Clear();
+ pz.Clear();
+ hz.Header = new byte[68000];
+
+ this.ReadToken("comp");
+ hz.Header[2] = (byte)this.ReadNumber(0, 255);
+ hz.Header[3] = (byte)this.ReadNumber(0, 255);
+ hz.Header[4] = (byte)this.ReadNumber(0, 255);
+ hz.Header[5] = (byte)this.ReadNumber(0, 255);
+ var n = this.ReadNumber(0, 255);
+ hz.Header[6] = (byte)n;
+ hz.Cend = 7;
+ for (var i = 0; i < n; ++i) {
+ this.ReadNumber(i, i);
+ var type = this.ReadToken(ComponentNames);
+ hz.Header[hz.Cend++] = (byte)type;
+ var clen = type < ZpaqlMachine.ComponentSize.Length ? ZpaqlMachine.ComponentSize[type] : 0;
+ if (clen is < 1 or > 10)
+ this.SyntaxError("invalid component");
+ for (var j = 1; j < clen; ++j)
+ hz.Header[hz.Cend++] = (byte)this.ReadNumber(0, 255);
+ }
+ ++hz.Cend;
+ hz.Hbegin = hz.Hend = hz.Cend + 128;
+
+ this.ReadToken("hcomp");
+ var op = this.CompileComp(hz);
+
+ var hsize = hz.Cend - 2 + hz.Hend - hz.Hbegin;
+ hz.Header[0] = (byte)hsize;
+ hz.Header[1] = (byte)(hsize >> 8);
+
+ if (op == Post) {
+ this.ReadNumber(0, 0);
+ this.ReadToken("end");
+ } else if (op == Pcomp) {
+ pz.Header = new byte[68000];
+ pz.Header[4] = hz.Header[4];
+ pz.Header[5] = hz.Header[5];
+ pz.Cend = 8;
+ pz.Hbegin = pz.Hend = pz.Cend + 128;
+
+ this.Next();
+ var start = this._pos;
+ while (this._pos < this._in.Length && this._in[this._pos] != ';')
+ ++this._pos;
+ this.PcompCommand = this._in[start..this._pos];
+ if (this._pos < this._in.Length)
+ ++this._pos;
+
+ op = this.CompileComp(pz);
+ var len = pz.Cend - 2 + pz.Hend - pz.Hbegin;
+ pz.Header[0] = (byte)len;
+ pz.Header[1] = (byte)(len >> 8);
+ if (op != End)
+ this.SyntaxError("expected END");
+ } else if (op != End)
+ this.SyntaxError("expected END or POST 0 END or PCOMP cmd ; ... END");
+ }
+
+ private char Cur => this._pos < this._in.Length ? this._in[this._pos] : '\0';
+
+ private char At(int offset) => this._pos + offset < this._in.Length ? this._in[this._pos + offset] : '\0';
+
+ private void Next() {
+ for (; this._pos < this._in.Length; ++this._pos) {
+ var ch = this._in[this._pos];
+ if (ch == '\n')
+ ++this._line;
+ if (ch == '(')
+ this._state += 1 + (this._state < 0 ? 1 : 0);
+ else if (this._state > 0 && ch == ')')
+ --this._state;
+ else if (this._state < 0 && ch <= ' ')
+ this._state = 0;
+ else if (this._state == 0 && ch > ' ') {
+ this._state = -1;
+ break;
+ }
+ }
+ if (this._pos >= this._in.Length)
+ ZpaqError.Throw("unexpected end of config");
+ }
+
+ private static char Lower(char c) => c is >= 'A' and <= 'Z' ? (char)(c + 'a' - 'A') : c;
+
+ private bool MatchToken(string word) {
+ var a = this._pos;
+ var w = 0;
+ for (; a < this._in.Length && this._in[a] > ' ' && this._in[a] != '(' && w < word.Length; ++a, ++w)
+ if (Lower(this._in[a]) != Lower(word[w]))
+ return false;
+ var next = a < this._in.Length ? this._in[a] : '\0';
+ return w == word.Length && (next <= ' ' || next == '(');
+ }
+
+ [System.Diagnostics.CodeAnalysis.DoesNotReturn]
+ private void SyntaxError(string message, string? expected = null) {
+ var token = new string(this._in.AsSpan(this._pos).ToArray().TakeWhile(c => c > ' ').Take(40).ToArray());
+ ZpaqError.Throw($"config line {this._line} at {token}: {message}" + (expected != null ? ", expected: " + expected : ""));
+ }
+
+ private int ReadToken(string?[] list) {
+ this.Next();
+ for (var i = 0; list[i] != null; ++i)
+ if (this.MatchToken(list[i]!))
+ return i;
+ this.SyntaxError("unexpected");
+ return -1;
+ }
+
+ private void ReadToken(string s) {
+ this.Next();
+ if (!this.MatchToken(s))
+ this.SyntaxError("expected", s);
+ }
+
+ private int ReadNumber(int low, int high) {
+ this.Next();
+ var r = 0;
+ if (this.Cur == '$' && this.At(1) is >= '1' and <= '9') {
+ if (this.At(2) == '+')
+ r = Atoi(this._in, this._pos + 3);
+ r += this._args[this.At(1) - '1'];
+ } else if (this.Cur == '-' || char.IsAsciiDigit(this.Cur))
+ r = Atoi(this._in, this._pos);
+ else
+ this.SyntaxError("expected a number");
+ if (r < low)
+ this.SyntaxError("number too low");
+ if (r > high)
+ this.SyntaxError("number too high");
+ return r;
+ }
+
+ /// C atoi: optional sign, then decimal digits up to the first non-digit.
+ private static int Atoi(string s, int i) {
+ var negative = false;
+ if (i < s.Length && (s[i] == '-' || s[i] == '+')) {
+ negative = s[i] == '-';
+ ++i;
+ }
+ var r = 0;
+ for (; i < s.Length && char.IsAsciiDigit(s[i]); ++i)
+ r = r * 10 + s[i] - '0';
+ return negative ? -r : r;
+ }
+
+ private int CompileComp(ZpaqlMachine z) {
+ int op;
+ var compBegin = z.Hend;
+ var header = z.Header;
+ while (true) {
+ op = this.ReadToken(OpcodeList);
+ if (op is Post or Pcomp or End)
+ break;
+ var operand = -1;
+ var operand2 = -1;
+ if (op == If) {
+ op = Jf;
+ operand = 0;
+ this._ifStack.Push(z.Hend + 1);
+ } else if (op == IfNot) {
+ op = Jt;
+ operand = 0;
+ this._ifStack.Push(z.Hend + 1);
+ } else if (op is IfL or IfNotL) {
+ if (op == IfL)
+ header[z.Hend++] = Jt;
+ if (op == IfNotL)
+ header[z.Hend++] = Jf;
+ header[z.Hend++] = 3;
+ op = Lj;
+ operand = operand2 = 0;
+ this._ifStack.Push(z.Hend + 1);
+ } else if (op is Else or ElseL) {
+ if (op == Else) {
+ op = Jmp;
+ operand = 0;
+ }
+ if (op == ElseL) {
+ op = Lj;
+ operand = operand2 = 0;
+ }
+ var a = this.PopIf();
+ if (header[a - 1] != Lj) {
+ var j = z.Hend - a + 1 + (op == Lj ? 1 : 0);
+ if (j > 127)
+ this.SyntaxError("IF too big, try IFL, IFNOTL");
+ header[a] = (byte)j;
+ } else {
+ var j = z.Hend - compBegin + 2 + (op == Lj ? 1 : 0);
+ header[a] = (byte)j;
+ header[a + 1] = (byte)(j >> 8);
+ }
+ this._ifStack.Push(z.Hend + 1);
+ } else if (op == EndIf) {
+ var a = this.PopIf();
+ var j = z.Hend - a - 1;
+ if (header[a - 1] != Lj) {
+ if (j > 127)
+ this.SyntaxError("IF too big, try IFL, IFNOTL, ELSEL");
+ header[a] = (byte)j;
+ } else {
+ j = z.Hend - compBegin;
+ header[a] = (byte)j;
+ header[a + 1] = (byte)(j >> 8);
+ }
+ } else if (op == Do)
+ this._doStack.Push(z.Hend);
+ else if (op is While or Until or Forever) {
+ if (this._doStack.Count == 0)
+ ZpaqError.Throw("unmatched IF or DO");
+ var a = this._doStack.Pop();
+ var j = a - z.Hend - 2;
+ if (j >= -127) {
+ op = op switch { While => Jt, Until => Jf, _ => Jmp };
+ operand = j & 255;
+ } else {
+ j = a - compBegin;
+ if (op == While) {
+ header[z.Hend++] = Jf;
+ header[z.Hend++] = 3;
+ }
+ if (op == Until) {
+ header[z.Hend++] = Jt;
+ header[z.Hend++] = 3;
+ }
+ op = Lj;
+ operand = j & 255;
+ operand2 = j >> 8;
+ }
+ } else if ((op & 7) == 7) {
+ if (op == Lj) {
+ operand = this.ReadNumber(0, 65535);
+ operand2 = operand >> 8;
+ operand &= 255;
+ } else if (op is Jt or Jf or Jmp) {
+ operand = this.ReadNumber(-128, 127);
+ operand &= 255;
+ } else
+ operand = this.ReadNumber(0, 255);
+ }
+ if (op is >= 0 and <= 255)
+ header[z.Hend++] = (byte)op;
+ if (operand >= 0)
+ header[z.Hend++] = (byte)operand;
+ if (operand2 >= 0)
+ header[z.Hend++] = (byte)operand2;
+ if (z.Hend >= header.Length - 130 || z.Hend - z.Hbegin + z.Cend - 2 > 65535)
+ this.SyntaxError("program too big");
+ }
+ header[z.Hend++] = 0;
+ return op;
+ }
+
+ private int PopIf() {
+ if (this._ifStack.Count == 0)
+ ZpaqError.Throw("unmatched IF or DO");
+ return this._ifStack.Pop();
+ }
+}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqCompressor.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqCompressor.cs
new file mode 100644
index 000000000..bbcf886d6
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqCompressor.cs
@@ -0,0 +1,103 @@
+using System.Text;
+
+namespace FileFormat.Zpaq;
+
+///
+/// Writes ZPAQ blocks: locator tag, block header (COMP + HCOMP), segments with filename,
+/// comment, the PCOMP program, the coded data and the SHA-1 trailer.
+///
+/// Converted from libzpaq 7.15 (Matt Mahoney, public domain), class Compressor.
+internal sealed class ZpaqCompressor {
+ private enum Phase { Init, Block1, Segment1, Block2, Segment2 }
+
+ private readonly ZpaqlMachine _z = new();
+ private readonly ZpaqlMachine _pz = new();
+ private readonly ZpaqEncoder _enc;
+ private Phase _state = Phase.Init;
+
+ /// The 13-byte locator tag that lets a reader find blocks after arbitrary leading data.
+ internal static ReadOnlySpan Tag => [0x37, 0x6b, 0x53, 0x74, 0xa0, 0x31, 0x83, 0xd3, 0x8c, 0xb2, 0x28, 0xb0, 0xd3];
+
+ /// Creates a compressor writing to .
+ public ZpaqCompressor(IZpaqWriter output) => this._enc = new(this._z) { Out = output };
+
+ /// Writes the locator tag.
+ public void WriteTag() => this._enc.Out.Write(Tag);
+
+ /// Starts a block from ZPAQL source ; returns the PCOMP command name.
+ public string StartBlock(string config, int[]? args) {
+ var compiler = new ZpaqCompiler(config, args, this._z, this._pz);
+ var output = this._enc.Out;
+ output.Put('z');
+ output.Put('P');
+ output.Put('Q');
+ output.Put(1 + (this._z.Header[6] == 0 ? 1 : 0));
+ output.Put(1);
+ this._z.Write(output, false);
+ this._state = Phase.Block1;
+ return compiler.PcompCommand;
+ }
+
+ /// Starts a segment carrying and .
+ public void StartSegment(string? filename, ReadOnlySpan comment) {
+ var output = this._enc.Out;
+ output.Put(1);
+ if (!string.IsNullOrEmpty(filename))
+ output.Write(Encoding.UTF8.GetBytes(filename));
+ output.Put(0);
+ output.Write(comment);
+ output.Put(0);
+ output.Put(0);
+ this._state = this._state == Phase.Block1 ? Phase.Segment1 : Phase.Segment2;
+ }
+
+ /// Codes all bytes from .
+ public void Compress(IZpaqReader input) {
+ if (this._state == Phase.Segment1)
+ this.PostProcess();
+ Span buf = stackalloc byte[1 << 14];
+ int nr;
+ while ((nr = input.Read(buf)) > 0)
+ for (var i = 0; i < nr; ++i)
+ this._enc.Compress(buf[i]);
+ }
+
+ /// Ends the segment, storing when given.
+ public void EndSegment(byte[]? sha1) {
+ if (this._state == Phase.Segment1)
+ this.PostProcess();
+ this._enc.Compress(-1);
+ var output = this._enc.Out;
+ output.Put(0);
+ output.Put(0);
+ output.Put(0);
+ output.Put(0);
+ if (sha1 != null) {
+ output.Put(253);
+ output.Write(sha1);
+ } else
+ output.Put(254);
+ this._state = Phase.Block2;
+ }
+
+ /// Ends the block.
+ public void EndBlock() {
+ this._enc.Out.Put(255);
+ this._state = Phase.Init;
+ }
+
+ /// Initialises coding and emits the PCOMP program (or the pass-through marker) at the start of the first segment.
+ private void PostProcess() {
+ this._enc.Init();
+ var len = this._pz.Hend - this._pz.Hbegin;
+ if (len > 0) {
+ this._enc.Compress(1);
+ this._enc.Compress(len & 255);
+ this._enc.Compress((len >> 8) & 255);
+ for (var i = 0; i < len; ++i)
+ this._enc.Compress(this._pz.Header[this._pz.Hbegin + i]);
+ } else
+ this._enc.Compress(0);
+ this._state = Phase.Segment2;
+ }
+}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqConstants.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqConstants.cs
deleted file mode 100644
index 37433ae30..000000000
--- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqConstants.cs
+++ /dev/null
@@ -1,62 +0,0 @@
-namespace FileFormat.Zpaq;
-
-///
-/// Constants for the ZPAQ archive format (level 1 journaling).
-///
-public static class ZpaqConstants {
- ///
- /// The 3-byte block locator prefix present at the start of every ZPAQ block ("zPQ").
- ///
- public static ReadOnlySpan BlockPrefix => "zPQ"u8;
-
- ///
- /// Compression level byte for ZPAQ level 1 archives.
- ///
- public const byte Level1 = 1;
-
- ///
- /// Compression level byte for ZPAQ level 2 archives.
- ///
- public const byte Level2 = 2;
-
- ///
- /// Block type byte: compressed data block (contains ZPAQL program + compressed payload).
- ///
- public const byte BlockTypeCompressed = 1;
-
- ///
- /// Block type byte: comment/filename header block in a journaling transaction.
- /// In a level-1 journaling archive this block carries the transaction date,
- /// filenames and per-file attributes for one transaction.
- ///
- public const byte BlockTypeHeader = (byte)'c';
-
- ///
- /// Block type byte: data block in a journaling transaction.
- /// Each data block holds the (ZPAQL-compressed) payload for one or more files.
- ///
- public const byte BlockTypeData = (byte)'d';
-
- ///
- /// Block type byte: hash/index block that closes a journaling transaction.
- /// Contains SHA-1 hashes of the uncompressed file data.
- ///
- public const byte BlockTypeIndex = (byte)'h';
-
- ///
- /// Minimum number of bytes in a valid block header
- /// (3-byte prefix + 1 level byte + 1 type byte = 5 bytes).
- ///
- public const int MinBlockHeaderSize = 5;
-
- ///
- /// Size of the Windows FILETIME timestamp stored in journaling header blocks (8 bytes).
- ///
- public const int FileTimeSize = 8;
-
- ///
- /// Windows FILETIME ticks (100-nanosecond intervals) from the Windows epoch
- /// (1601-01-01) to the Unix epoch (1970-01-01), used when converting timestamps.
- ///
- public const long WindowsToUnixEpochTicks = 116444736000000000L;
-}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqDecompresser.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqDecompresser.cs
new file mode 100644
index 000000000..2ff7015b2
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqDecompresser.cs
@@ -0,0 +1,156 @@
+using System.Text;
+
+namespace FileFormat.Zpaq;
+
+///
+/// Walks a ZPAQ stream block by block and segment by segment: locate a block, read its
+/// segments' filename and comment, decode the data, read the trailing SHA-1.
+///
+/// Converted from libzpaq 7.15 (Matt Mahoney, public domain), class Decompresser.
+internal sealed class ZpaqDecompresser {
+ private enum Expect { Block, Filename, Comment, Data, SegmentEnd }
+ private enum DecodeState { FirstSegment, Segment, Skip }
+
+ private readonly ZpaqlMachine _z = new();
+ private readonly ZpaqDecoder _dec;
+ private readonly ZpaqPostProcessor _pp = new();
+ private Expect _state = Expect.Block;
+ private DecodeState _decodeState = DecodeState.FirstSegment;
+
+ /// Creates a decompresser reading from .
+ public ZpaqDecompresser(IZpaqReader input) {
+ this._dec = new(this._z) { In = input };
+ }
+
+ /// Bytes read ahead of the logical position.
+ public int Buffered => this._dec.Buffered;
+
+ /// Level byte (1 or 2) of the current block.
+ public int Level { get; private set; }
+
+ /// Approximate memory the current block needs to decode.
+ public double Memory => this._z.Memory();
+
+ /// Discards read-ahead after the caller repositioned the underlying input.
+ public void DiscardBuffer() => this._dec.DiscardBuffer();
+
+ /// Scans for the next block header and reads it; at end of input.
+ public bool FindBlock() {
+ uint h1 = 0x3D49B113, h2 = 0x29EB7F93, h3 = 0x2614BE13, h4 = 0x3828EB13;
+ int c;
+ while ((c = this._dec.Get()) != -1) {
+ h1 = h1 * 12 + (uint)c;
+ h2 = h2 * 20 + (uint)c;
+ h3 = h3 * 28 + (uint)c;
+ h4 = h4 * 44 + (uint)c;
+ if (h1 == 0xB16B88F1 && h2 == 0xFF5376F1 && h3 == 0x72AC5BF1 && h4 == 0x2F909AF1)
+ break;
+ }
+ if (c == -1)
+ return false;
+
+ if ((c = this._dec.Get()) != 1 && c != 2)
+ ZpaqError.Throw("unsupported ZPAQ level");
+ if (this._dec.Get() != 1)
+ ZpaqError.Throw("unsupported ZPAQL type");
+ this._z.Read(this._dec);
+ if (c == 1 && this._z.Header.Length > 6 && this._z.Header[6] == 0)
+ ZpaqError.Throw("ZPAQ level 1 requires at least 1 component");
+ this.Level = c;
+ this._state = Expect.Filename;
+ this._decodeState = DecodeState.FirstSegment;
+ return true;
+ }
+
+ /// Reads the next segment header; returns its filename, or at end of block.
+ public string? FindFilename() {
+ var c = this._dec.Get();
+ if (c == 1) {
+ var name = new List();
+ while (true) {
+ c = this._dec.Get();
+ if (c == -1)
+ ZpaqError.Throw("unexpected EOF");
+ if (c == 0) {
+ this._state = Expect.Comment;
+ return Encoding.UTF8.GetString(name.ToArray());
+ }
+ name.Add((byte)c);
+ }
+ }
+ if (c == 255) {
+ this._state = Expect.Block;
+ return null;
+ }
+ ZpaqError.Throw("missing segment or end of block");
+ return null;
+ }
+
+ /// Reads the comment of the current segment as raw bytes.
+ public byte[] ReadComment() {
+ this._state = Expect.Data;
+ var comment = new List();
+ while (true) {
+ var c = this._dec.Get();
+ if (c == -1)
+ ZpaqError.Throw("unexpected EOF");
+ if (c == 0)
+ break;
+ comment.Add((byte)c);
+ }
+ if (this._dec.Get() != 0)
+ ZpaqError.Throw("missing reserved byte");
+ return [.. comment];
+ }
+
+ /// Sets where decoded (post-processed) bytes go; discards them.
+ public void SetOutput(IZpaqWriter? output) => this._pp.Z.Output = output;
+
+ /// Sets the hash that sees every output byte, or .
+ public void SetSha1(ZpaqSha1? sha1) => this._pp.Z.Sha1 = sha1;
+
+ /// Decodes the whole current segment.
+ public void Decompress() {
+ if (this._decodeState == DecodeState.Skip)
+ ZpaqError.Throw("decompression after skipped segment");
+ if (this._decodeState == DecodeState.FirstSegment) {
+ this._dec.Init();
+ this._pp.Init(this._z.Header[4], this._z.Header[5]);
+ this._decodeState = DecodeState.Segment;
+ }
+ while ((this._pp.State & 3) != 1)
+ this._pp.Write(this._dec.Decompress());
+ while (true) {
+ var c = this._dec.Decompress();
+ this._pp.Write(c);
+ if (c == -1) {
+ this._state = Expect.SegmentEnd;
+ return;
+ }
+ }
+ }
+
+ /// Finishes the segment, skipping undecoded data; returns the stored SHA-1 or .
+ public byte[]? ReadSegmentEnd() {
+ var c = 0;
+ if (this._state == Expect.Data) {
+ c = this._dec.Skip();
+ this._decodeState = DecodeState.Skip;
+ } else if (this._state == Expect.SegmentEnd)
+ c = this._dec.Get();
+ this._state = Expect.Filename;
+
+ if (c == 254)
+ return null;
+ if (c != 253)
+ ZpaqError.Throw("missing end of segment marker");
+ var hash = new byte[20];
+ for (var i = 0; i < 20; ++i) {
+ c = this._dec.Get();
+ if (c < 0)
+ ZpaqError.Throw("unexpected EOF");
+ hash[i] = (byte)c;
+ }
+ return hash;
+ }
+}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqEntry.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqEntry.cs
index a52164491..18d45895d 100644
--- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqEntry.cs
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqEntry.cs
@@ -1,59 +1,52 @@
namespace FileFormat.Zpaq;
///
-/// Represents a single file entry recorded in a ZPAQ journaling archive.
+/// A file or directory as recorded in a ZPAQ archive.
///
public sealed class ZpaqEntry {
- ///
- /// Gets the filename as stored in the archive.
- /// Path separators are always forward slashes; directory entries end with '/'.
- ///
+ /// Path as stored, with forward slashes; directories end with '/'.
public string FileName { get; }
- ///
- /// Gets the uncompressed size of the file in bytes as reported by the archive.
- /// This value is zero for directory entries and for files whose size could not
- /// be determined from the journal (e.g. when the header block is compressed).
- ///
+ /// Uncompressed size in bytes, or -1 when the archive does not record it (streaming format).
public long Size { get; }
///
- /// Gets the compressed size of the data block(s) that carry this file's data.
- /// This is the total byte count of the raw (still-compressed) data blocks
- /// associated with this entry.
+ /// Compressed size attributed to this entry: the share of the data blocks it references,
+ /// proportional to its new (non-deduplicated) fragments. Zero for directories.
///
public long CompressedSize { get; }
- ///
- /// Gets the last-modified timestamp decoded from the journal header, or
- /// when the timestamp could not be recovered.
- ///
+ /// Last-modified time (UTC) from the archive's decimal date, or .
public DateTime? LastModified { get; }
- ///
- /// Gets a value indicating whether this entry represents a directory.
- ///
+ /// Whether this entry is a directory.
public bool IsDirectory { get; }
+ /// 1-based version (transaction) that last wrote this entry.
+ public int Version { get; }
+
///
- /// Gets the zero-based transaction index in which this entry was recorded.
- /// Each append to a ZPAQ archive constitutes a new transaction; the most
- /// recent transaction with a given filename defines the file's current state.
+ /// Raw attribute bytes as stored (first byte 'w' = Windows attributes follow, 'u' = Unix mode
+ /// follows), packed little-endian; 0 when none were stored.
///
- public int Version { get; }
+ public long Attributes { get; }
+
+ /// Fragment IDs that make up the file (journaling format).
+ internal IReadOnlyList Fragments { get; }
+
+ /// Streaming-format segments (block offset, segment index) that make up the file.
+ internal IReadOnlyList<(long BlockOffset, int Segment)> Segments { get; }
- internal ZpaqEntry(
- string fileName,
- long size,
- long compressedSize,
- DateTime? lastModified,
- bool isDirectory,
- int version) {
- FileName = fileName;
- Size = size;
- CompressedSize = compressedSize;
- LastModified = lastModified;
- IsDirectory = isDirectory;
- Version = version;
+ internal ZpaqEntry(string fileName, long size, long compressedSize, DateTime? lastModified, int version, long attributes,
+ IReadOnlyList fragments, IReadOnlyList<(long, int)> segments) {
+ this.FileName = fileName;
+ this.Size = size;
+ this.CompressedSize = compressedSize;
+ this.LastModified = lastModified;
+ this.IsDirectory = fileName.EndsWith('/');
+ this.Version = version;
+ this.Attributes = attributes;
+ this.Fragments = fragments;
+ this.Segments = segments;
}
}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqFormatDescriptor.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqFormatDescriptor.cs
index 0dea9ff09..ef1e90d74 100644
--- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqFormatDescriptor.cs
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqFormatDescriptor.cs
@@ -1,145 +1,176 @@
#pragma warning disable CS1591
+using System.Globalization;
+using System.Text;
using Compression.Registry;
using static Compression.Registry.FormatHelpers;
namespace FileFormat.Zpaq;
///
-/// ZPAQ journaling archive with content-defined deduplication and configurable context-mixing compression.
+/// ZPAQ archive (journaling and streaming layouts) with deduplication and ZPAQL-described
+/// context-mixing compression.
///
/// References:
///
-/// - http://mattmahoney.net/dc/zpaq.html — official ZPAQ page and Level 2 format specification (Matt Mahoney)
-/// - https://github.com/zpaq/zpaq — reference implementation
-/// - https://en.wikipedia.org/wiki/ZPAQ — Wikipedia overview
+/// - https://mattmahoney.net/dc/zpaq206.pdf — ZPAQ Level 2 specification (Matt Mahoney)
+/// - https://mattmahoney.net/dc/zpaq.html — zpaq / libzpaq reference implementation (public domain)
///
///
public sealed class ZpaqFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveModifiable, IArchiveDefragmentable {
///
- /// A ZPAQ stream is a sequence of blocks; a file carrying none is not a ZPAQ archive, so there is no empty instance to purge to.
+ /// zpaq truncates an update that adds nothing, so there is no empty archive to purge to.
public bool CanPurgeToEmpty => false;
-
- /// Rebuild-based defrag: extracts then re-creates the ZPAQ archive in listing order.
+ /// Rebuild-based defrag: extracts then re-creates the archive as a single version.
public void Defragment(Stream archive)
=> this.Defragment(archive, new DefragOptions { Mode = DefragMode.ConsolidateAtStart });
- /// Rebuild-based defrag: extracts then re-creates the ZPAQ archive per the requested mode.
+ /// Rebuild-based defrag: extracts then re-creates the archive as a single version.
public void Defragment(Stream archive, DefragOptions options) {
DefragRebuilder.Rebuild(archive, options,
readEntries: stream => {
- var r = new ZpaqReader(stream);
- var list = new List<(string, byte[])>();
- foreach (var e in r.Entries) {
- if (e.IsDirectory) continue;
- using var es = r.Extract(e);
- using var ms = new MemoryStream();
- es.CopyTo(ms);
- list.Add((e.FileName, ms.ToArray()));
- }
- return list;
+ using var r = new ZpaqReader(stream, leaveOpen: true);
+ return r.Entries.Where(e => !e.IsDirectory).Select(e => (e.FileName, r.ExtractBytes(e))).ToList();
},
buildImage: files => {
using var ms = new MemoryStream();
- using (var w = new ZpaqWriter(ms, leaveOpen: true)) {
- foreach (var (n, d) in files) w.AddFile(n, d);
- }
+ using (var w = new ZpaqWriter(ms, leaveOpen: true))
+ foreach (var (n, d) in files)
+ w.AddFile(n, d);
return ms.ToArray();
});
}
- ///
- /// Gets the id.
- ///
public string Id => "Zpaq";
- ///
- /// Gets the display name.
- ///
public string DisplayName => "ZPAQ";
- ///
- /// Gets the category.
- ///
public FormatCategory Category => FormatCategory.Archive;
- // R/W: a mutable archive. Add/Replace/Remove go through the verified extract ->
- // edit -> re-create rebuild (default IArchiveModifiable); relayouting the container
- // on edit is honest R/W. See FormatCapabilities.cs (WORM vs R/W).
- ///
- /// Gets the capabilities.
- ///
+
+ // R/W: zpaq 7.15 lists and extracts what ZpaqWriter produces (ZpaqExternalOracleTests), and the
+ // reader decodes zpaq's own archives at every method. Modify goes through the default
+ // extract -> edit -> re-create rebuild, which writes a fresh single-version archive.
public FormatCapabilities Capabilities =>
FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate |
FormatCapabilities.CanModify |
FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories;
- ///
- /// Gets the default extension.
- ///
+
public string DefaultExtension => ".zpaq";
- ///
- /// Gets the extensions.
- ///
public IReadOnlyList Extensions => [".zpaq"];
- ///
- /// Gets the compound extensions.
- ///
public IReadOnlyList CompoundExtensions => [];
- ///
- /// Gets the magic signatures.
- ///
- public IReadOnlyList MagicSignatures => [new([(byte)'z', (byte)'P', (byte)'Q'], Confidence: 0.90)];
- ///
- /// Gets the methods.
- ///
- public IReadOnlyList Methods => [new("zpaq", "ZPAQ")];
- ///
- /// Gets the tar compression format id.
- ///
+
+ // Journaling archives and libzpaq streams start with the 13-byte locator tag "7kSt\xA0\x31\x83\xD3\x8C\xB2\x28\xB0\xD3";
+ // tagless streams start directly with the "zPQ" block header.
+ public IReadOnlyList MagicSignatures => [
+ new([0x37, 0x6B, 0x53, 0x74, 0xA0, 0x31, 0x83, 0xD3, 0x8C, 0xB2, 0x28, 0xB0, 0xD3], Confidence: 0.95),
+ new([(byte)'z', (byte)'P', (byte)'Q'], Confidence: 0.80),
+ ];
+
+ public IReadOnlyList Methods => [
+ new("1", "ZPAQ method 1 (LZ77, default)"),
+ new("0", "ZPAQ method 0 (dedup only)"),
+ new("2", "ZPAQ method 2 (LZ77, slower)"),
+ new("3", "ZPAQ method 3 (BWT / LZ77+CM)"),
+ new("4", "ZPAQ method 4 (CM)"),
+ new("5", "ZPAQ method 5 (CM, slowest)"),
+ ];
+
public string? TarCompressionFormatId => null;
- ///
- /// Gets the family.
- ///
public AlgorithmFamily Family => AlgorithmFamily.Archive;
- ///
- /// Gets the description.
- ///
- public string Description => "ZPAQ journaling archive with deduplication";
-
- ///
- /// Lists the entries in the supplied container.
- ///
+ public string Description => "ZPAQ journaling archive with deduplication and ZPAQL context-mixing compression";
+
public List List(Stream stream, string? password) {
- var r = new ZpaqReader(stream);
- return r.Entries.Select((e, i) => new ArchiveEntryInfo(i, e.FileName, e.Size, e.CompressedSize,
- "zpaq", e.IsDirectory, false, e.LastModified)).ToList();
+ var seekable = Seekable(stream);
+ try {
+ var (reader, _) = TryOpen(seekable);
+ if (reader == null) {
+ var length = seekable.Length;
+ return [
+ new(0, "FULL.zpaq", length, length, "Stored", false, false, null, Kind: "Track"),
+ new(1, "metadata.ini", 0, 0, "Stored", false, false, null, Kind: "Tag"),
+ ];
+ }
+ using (reader)
+ return reader.Entries.Select((e, i) => new ArchiveEntryInfo(i, e.FileName, Math.Max(e.Size, 0), e.CompressedSize,
+ "zpaq", e.IsDirectory, false, e.LastModified)).ToList();
+ } finally {
+ if (!ReferenceEquals(seekable, stream))
+ seekable.Dispose();
+ }
}
- ///
- /// Decodes the supplied input.
- ///
public void Extract(Stream stream, string outputDir, string? password, string[]? files) {
- var r = new ZpaqReader(stream);
- foreach (var e in r.Entries) {
- if (e.IsDirectory) continue;
- if (files != null && !MatchesFilter(e.FileName, files)) continue;
- using var es = r.Extract(e);
- using var ms = new MemoryStream();
- es.CopyTo(ms);
- WriteFile(outputDir, e.FileName, ms.ToArray());
+ var seekable = Seekable(stream);
+ try {
+ var (reader, problem) = TryOpen(seekable);
+ if (reader == null) {
+ if (Wants(files, "FULL.zpaq")) {
+ seekable.Position = 0;
+ using var ms = new MemoryStream();
+ seekable.CopyTo(ms);
+ WriteFile(outputDir, "FULL.zpaq", ms.ToArray());
+ }
+ if (Wants(files, "metadata.ini"))
+ WriteFile(outputDir, "metadata.ini", Encoding.UTF8.GetBytes(
+ "[Zpaq]\n" + string.Create(CultureInfo.InvariantCulture, $"problem={problem?.Replace('\n', ' ')}\n") +
+ "member_enumeration=unavailable\nparse_status=partial\n"));
+ return;
+ }
+ using (reader) {
+ // Visit files in fragment order so consecutive files reuse the decoded block.
+ var ordered = reader.Entries
+ .Where(e => !e.IsDirectory && (files == null || MatchesFilter(e.FileName, files)))
+ .OrderBy(e => e.Fragments.Count > 0 ? e.Fragments[0] : 0u);
+ foreach (var e in ordered)
+ WriteFile(outputDir, e.FileName, reader.ExtractBytes(e));
+ }
+ } finally {
+ if (!ReferenceEquals(seekable, stream))
+ seekable.Dispose();
}
}
- ///
- /// Performs the create operation.
- ///
public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) {
- using var w = new ZpaqWriter(output, leaveOpen: true);
+ ArgumentNullException.ThrowIfNull(output);
+ ArgumentNullException.ThrowIfNull(inputs);
+ ArgumentNullException.ThrowIfNull(options);
+ using var w = new ZpaqWriter(output, leaveOpen: true, MethodFor(options));
foreach (var input in inputs) {
- if (input.IsDirectory) {
+ if (input.IsDirectory)
w.AddDirectory(input.ArchiveName);
- } else {
- var data = input.ReadContent();
- w.AddFile(input.ArchiveName, data);
- }
+ else
+ w.AddFile(input.ArchiveName, input.ReadContent());
+ }
+ }
+
+ /// Maps the create options to a zpaq method: an explicit method string, else the level 0..5, else 1.
+ internal static string MethodFor(FormatCreateOptions options) {
+ var m = options.MethodName;
+ if (!string.IsNullOrEmpty(m) && (m[0] is >= '0' and <= '5' || m[0] == 'x'))
+ return m;
+ if (options.Level is { } level)
+ return Math.Clamp(level, 0, 5).ToString(CultureInfo.InvariantCulture);
+ return "1";
+ }
+
+ private static (ZpaqReader? Reader, string? Problem) TryOpen(Stream stream) {
+ try {
+ stream.Position = 0;
+ return (new ZpaqReader(stream, leaveOpen: true), null);
+ } catch (Exception e) when (e is InvalidDataException or NotSupportedException or EndOfStreamException or IOException
+ or OverflowException or ArgumentException or IndexOutOfRangeException) {
+ return (null, e.Message);
}
}
+
+ private static Stream Seekable(Stream stream) {
+ if (stream.CanSeek)
+ return stream;
+ var ms = new MemoryStream();
+ stream.CopyTo(ms);
+ ms.Position = 0;
+ return ms;
+ }
+
+ private static bool Wants(string[]? files, string name)
+ => files == null || files.Length == 0 || MatchesFilter(name, files);
}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqIo.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqIo.cs
new file mode 100644
index 000000000..2165d4f77
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqIo.cs
@@ -0,0 +1,186 @@
+using Compression.Core.Checksums;
+
+namespace FileFormat.Zpaq;
+
+/// Byte source consumed by the ZPAQ engine.
+internal interface IZpaqReader {
+ /// Returns the next byte (0..255) or -1 at end of input.
+ int Get();
+
+ /// Reads up to .Length bytes and returns the count; 0 means end of input.
+ int Read(Span buffer) {
+ var n = 0;
+ for (; n < buffer.Length; ++n) {
+ var c = this.Get();
+ if (c < 0)
+ break;
+ buffer[n] = (byte)c;
+ }
+ return n;
+ }
+}
+
+/// Byte sink fed by the ZPAQ engine.
+internal interface IZpaqWriter {
+ /// Writes the low eight bits of .
+ void Put(int c);
+
+ /// Writes a run of bytes.
+ void Write(ReadOnlySpan data) {
+ foreach (var b in data)
+ this.Put(b);
+ }
+}
+
+/// Growable in-memory byte buffer that can be appended to and read back, like libzpaq's StringBuffer.
+internal sealed class ZpaqBuffer : IZpaqReader, IZpaqWriter {
+ private byte[] _data;
+ private int _readPosition;
+
+ /// Creates an empty buffer with the given initial capacity.
+ public ZpaqBuffer(int capacity = 0) => this._data = capacity > 0 ? new byte[capacity] : [];
+
+ /// Creates a buffer that holds a copy of .
+ public ZpaqBuffer(ReadOnlySpan content) {
+ this._data = content.ToArray();
+ this.Size = content.Length;
+ }
+
+ /// Number of bytes written.
+ public int Size { get; private set; }
+
+ /// Maximum number of bytes that may be written before the buffer refuses more.
+ public long Limit { get; set; } = long.MaxValue;
+
+ /// Bytes written but not read yet.
+ public int Remaining => this.Size - this._readPosition;
+
+ /// The written content; the backing array may be longer than .
+ public byte[] Data => this._data;
+
+ /// The written content as a span.
+ public Span Span => this._data.AsSpan(0, this.Size);
+
+ ///
+ public int Get() => this._readPosition < this.Size ? this._data[this._readPosition++] : -1;
+
+ ///
+ public int Read(Span buffer) {
+ var n = Math.Min(buffer.Length, this.Size - this._readPosition);
+ if (n <= 0)
+ return 0;
+ this._data.AsSpan(this._readPosition, n).CopyTo(buffer);
+ this._readPosition += n;
+ return n;
+ }
+
+ ///
+ public void Put(int c) {
+ this.EnsureRoom(1);
+ this._data[this.Size++] = (byte)c;
+ }
+
+ ///
+ public void Write(ReadOnlySpan data) {
+ if (data.IsEmpty)
+ return;
+ this.EnsureRoom(data.Length);
+ data.CopyTo(this._data.AsSpan(this.Size));
+ this.Size += data.Length;
+ }
+
+ /// Truncates (or extends without clearing) the written size and clamps the read position.
+ public void Resize(int size) {
+ if (size > this._data.Length)
+ Array.Resize(ref this._data, size);
+ this.Size = size;
+ if (this._readPosition > size)
+ this._readPosition = size;
+ }
+
+ /// Discards all content and rewinds the read position.
+ public void Reset() {
+ this.Size = 0;
+ this._readPosition = 0;
+ }
+
+ /// Returns a copy of the written bytes.
+ public byte[] ToArray() => this._data.AsSpan(0, this.Size).ToArray();
+
+ private void EnsureRoom(int n) {
+ if (this.Size + (long)n > this.Limit)
+ ZpaqError.Throw("buffer overflow");
+ if (this.Size + n <= this._data.Length)
+ return;
+ var capacity = Math.Max(128, this._data.Length);
+ while (capacity < this.Size + n)
+ capacity = capacity > Array.MaxLength / 2 ? Array.MaxLength : capacity * 2;
+ Array.Resize(ref this._data, capacity);
+ }
+}
+
+/// Reads from a .
+internal sealed class ZpaqStreamReader(Stream stream) : IZpaqReader {
+ ///
+ public int Get() => stream.ReadByte();
+
+ ///
+ public int Read(Span buffer) => stream.Read(buffer);
+}
+
+/// Writes to a .
+internal sealed class ZpaqStreamWriter(Stream stream) : IZpaqWriter {
+ ///
+ public void Put(int c) => stream.WriteByte((byte)c);
+
+ ///
+ public void Write(ReadOnlySpan data) => stream.Write(data);
+}
+
+/// Discards everything written to it.
+internal sealed class ZpaqNullWriter : IZpaqWriter {
+ /// The shared instance.
+ public static readonly ZpaqNullWriter Instance = new();
+
+ ///
+ public void Put(int c) { }
+
+ ///
+ public void Write(ReadOnlySpan data) { }
+}
+
+/// Incremental SHA-1 that also counts the bytes hashed, matching libzpaq's SHA1 usage.
+internal sealed class ZpaqSha1 : IZpaqWriter {
+ private readonly Sha1 _sha1 = new();
+
+ /// Number of bytes hashed since the last .
+ public long Size { get; private set; }
+
+ ///
+ public void Put(int c) {
+ Span one = [(byte)c];
+ this.Write(one);
+ }
+
+ ///
+ public void Write(ReadOnlySpan data) {
+ this._sha1.Update(data);
+ this.Size += data.Length;
+ }
+
+ /// Returns the 20-byte digest and resets for the next input.
+ public byte[] Result() {
+ this._sha1.Finish();
+ var hash = this._sha1.Hash;
+ this._sha1.Reset();
+ this.Size = 0;
+ return hash;
+ }
+}
+
+/// Raises the engine's error for malformed input or an invalid model.
+internal static class ZpaqError {
+ /// Throws an carrying .
+ [System.Diagnostics.CodeAnalysis.DoesNotReturn]
+ public static void Throw(string message) => throw new InvalidDataException("ZPAQ: " + message);
+}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqLzBuffer.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqLzBuffer.cs
new file mode 100644
index 000000000..82434a765
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqLzBuffer.cs
@@ -0,0 +1,351 @@
+namespace FileFormat.Zpaq;
+
+///
+/// The LZ77 / BWT preprocessor of ZPAQ methods 1..3 (the x method's second argument),
+/// presented as a byte source the compressor reads.
+///
+///
+/// Converted from libzpaq 7.15 (Matt Mahoney, public domain), class LZBuffer.
+///
+/// Level 1 emits bit-packed LZ77 codes, level 2 byte-aligned LZ77 codes, level 3 the
+/// Burrows-Wheeler transform with the end-of-string position in the last four bytes. The
+/// matching PCOMP programs that undo each of these are generated by
+/// . The suffix array (for BWT and for the suffix-array
+/// match finder) comes from ; a suffix array is unique, so the
+/// output equals the reference's regardless of how it is built.
+///
+///
+internal sealed class ZpaqLzBuffer : IZpaqReader {
+ private const int BufSize = 1 << 14;
+
+ private readonly uint[] _ht;
+ private readonly byte[] _in;
+ private readonly int _checkbits;
+ private readonly int _level;
+ private readonly uint _htsize;
+ private readonly uint _n;
+ private uint _i;
+ private readonly uint _minMatch;
+ private readonly uint _minMatch2;
+ private readonly uint _maxMatch = BufSize * 3;
+ private readonly uint _maxLiteral = BufSize / 4;
+ private readonly uint _lookahead;
+ private uint _h1, _h2;
+ private readonly uint _bucket;
+ private readonly int _shift1, _shift2;
+ private readonly int _minMatchBoth;
+ private readonly int _rb;
+ private uint _bits;
+ private int _nbits;
+ private int _rpos, _wpos;
+ private uint _idx;
+ private readonly int[]? _sa;
+ private readonly int _isaOffset = -1;
+ private readonly byte[] _buf = new byte[BufSize];
+
+ /// Prepares to encode with method arguments .
+ public ZpaqLzBuffer(ZpaqBuffer input, int[] args) {
+ // A private copy with zero padding: the hash match finder peeks a few bytes past the
+ // end (the reference reads its buffer's slack there), and E8E9 must not touch the caller's data.
+ this._n = (uint)input.Size;
+ this._in = new byte[input.Size + 8];
+ input.Span.CopyTo(this._in);
+ this._level = args[1] & 3;
+ this._minMatch = (uint)args[2];
+ this._minMatch2 = (uint)args[3];
+ this._lookahead = (uint)args[6];
+ this._bucket = (1u << args[4]) - 1;
+ this._shift1 = this._minMatch > 0 ? (args[5] - 1) / (int)this._minMatch + 1 : 1;
+ this._shift2 = this._minMatch2 > 0 ? (args[5] - 1) / (int)this._minMatch2 + 1 : 0;
+ this._minMatchBoth = (int)Math.Max(this._minMatch, this._minMatch2 + this._lookahead) + 4;
+ this._rb = args[0] > 4 ? args[0] - 4 : 0;
+ var useSa = args[5] - args[0] >= 21;
+ this._checkbits = useSa ? 17 + args[0] : 12 - args[0];
+
+ if ((this._minMatch < 4 && this._level == 1) || (this._minMatch < 1 && this._level == 2))
+ ZpaqError.Throw("match length $3 too small");
+
+ if (args[1] > 4)
+ ZpaqMethods.E8E9(this._in.AsSpan(0, (int)this._n));
+
+ if (this._level == 3) {
+ this._ht = [];
+ this._sa = ZpaqSuffixArray.Build(this._in.AsSpan(0, (int)this._n));
+ } else if (useSa) {
+ this._sa = ZpaqSuffixArray.Build(this._in.AsSpan(0, (int)this._n));
+ this._ht = new uint[1u << 17 << args[0]];
+ this._isaOffset = 0;
+ } else
+ this._ht = new uint[1u << args[5]];
+ this._htsize = (uint)this._ht.Length;
+ }
+
+ ///
+ public int Get() {
+ var c = -1;
+ if (this._rpos == this._wpos)
+ this.Fill();
+ if (this._rpos < this._wpos)
+ c = this._buf[this._rpos++];
+ if (this._rpos == this._wpos)
+ this._rpos = this._wpos = 0;
+ return c;
+ }
+
+ ///
+ public int Read(Span p) {
+ if (this._rpos == this._wpos)
+ this.Fill();
+ var nr = Math.Min(p.Length, this._wpos - this._rpos);
+ this._buf.AsSpan(this._rpos, nr).CopyTo(p);
+ this._rpos += nr;
+ if (this._rpos == this._wpos)
+ this._rpos = this._wpos = 0;
+ return nr;
+ }
+
+ private static int Lg(uint x) => x == 0 ? 0 : 32 - System.Numerics.BitOperations.LeadingZeroCount(x);
+
+ private void PutBits(uint x, int k) {
+ x &= (uint)((1L << k) - 1);
+ this._bits |= x << this._nbits;
+ this._nbits += k;
+ while (this._nbits > 7) {
+ this._buf[this._wpos++] = (byte)this._bits;
+ this._bits >>= 8;
+ this._nbits -= 8;
+ }
+ }
+
+ private void FlushBits() {
+ if (this._nbits > 0)
+ this._buf[this._wpos++] = (byte)this._bits;
+ this._bits = 0;
+ this._nbits = 0;
+ }
+
+ private void Put(int c) => this._buf[this._wpos++] = (byte)c;
+
+ private void Fill() {
+ var input = this._in;
+ var n = this._n;
+
+ if (this._level == 3) {
+ var sa = this._sa!;
+ for (; this._wpos < BufSize && this._i < n + 5; ++this._i) {
+ if (this._i == 0)
+ this.Put(n > 0 ? input[n - 1] : 255);
+ else if (this._i > n) {
+ this.Put((int)(this._idx & 255));
+ this._idx >>= 8;
+ } else if (sa[this._i - 1] == 0) {
+ this._idx = this._i;
+ this.Put(255);
+ } else
+ this.Put(input[sa[this._i - 1] - 1]);
+ }
+ return;
+ }
+
+ var lit = 0u;
+ var mask = (uint)((1L << this._checkbits) - 1);
+ var ht = this._ht;
+ while (this._i < n && this._wpos * 2 < BufSize) {
+ var i = this._i;
+ var blen = this._minMatch - 1;
+ var bp = 0u;
+ var blit = 0u;
+ var bscore = 0;
+
+ if (this._isaOffset >= 0) {
+ var sa = this._sa!;
+ if ((uint)sa[ht[i & mask]] != i)
+ for (var j = 0u; j < n; ++j)
+ if (((uint)sa[j] & ~mask) == (i & ~mask))
+ ht[(uint)sa[j] & mask] = j;
+ for (var h = 0u; h <= this._lookahead; ++h) {
+ var q = ht[(h + i) & mask];
+ if ((uint)sa[q] != h + i)
+ continue;
+ for (var j = -1; j <= 1; j += 2) {
+ for (var k = 1u; k <= this._bucket; ++k) {
+ var qq = (uint)(q + j * (long)k);
+ uint p;
+ if (qq < n && (p = (uint)sa[qq] - h) < i) {
+ uint l, l1;
+ for (l = h; i + l < n && l < this._maxMatch && input[p + l] == input[i + l]; ++l) { }
+ for (l1 = h; l1 > 0 && input[p + l1 - 1] == input[i + l1 - 1]; --l1) { }
+ var score = (int)(l - l1) * 8 - Lg(i - p) - 4 * (lit == 0 && l1 > 0 ? 1 : 0) - 11;
+ for (var a = 0u; a < h; ++a)
+ score = score * 5 / 8;
+ if (score > bscore) {
+ blen = l;
+ bp = p;
+ blit = l1;
+ bscore = score;
+ }
+ if (l < blen || l < this._minMatch || l > 255)
+ break;
+ }
+ }
+ }
+ if (bscore <= 0 || blen < this._minMatch)
+ break;
+ }
+ } else if (this._level == 1 || this._minMatch <= 64) {
+ if (this._minMatch2 > 0) {
+ for (var k = 0u; k <= this._bucket; ++k) {
+ var p = ht[this._h2 ^ k];
+ if (p != 0 && (p & mask) == (input[i + 3] & mask)) {
+ p >>= this._checkbits;
+ if (p < i && i + blen <= n && (blen == 0 || input[p + blen - 1] == input[i + blen - 1])) {
+ uint l;
+ for (l = this._lookahead; i + l < n && l < this._maxMatch && input[p + l] == input[i + l]; ++l) { }
+ if (l >= this._minMatch2 + this._lookahead) {
+ int l1;
+ for (l1 = (int)this._lookahead; l1 > 0 && input[p + l1 - 1] == input[i + l1 - 1]; --l1) { }
+ var score = (int)(l - (uint)l1) * 8 - Lg(i - p) - 8 * (lit == 0 && l1 > 0 ? 1 : 0) - 11;
+ if (score > bscore) {
+ blen = l;
+ bp = p;
+ blit = (uint)l1;
+ bscore = score;
+ }
+ }
+ }
+ }
+ if (blen >= 128)
+ break;
+ }
+ }
+
+ if (this._minMatch2 == 0 || blen < this._minMatch2) {
+ for (var k = 0u; k <= this._bucket; ++k) {
+ var p = ht[this._h1 ^ k];
+ if (p != 0 && i + 3 < n && (p & mask) == (input[i + 3] & mask)) {
+ p >>= this._checkbits;
+ if (p < i && i + blen <= n && (blen == 0 || input[p + blen - 1] == input[i + blen - 1])) {
+ uint l;
+ for (l = 0; i + l < n && l < this._maxMatch && input[p + l] == input[i + l]; ++l) { }
+ var score = (int)l * 8 - Lg(i - p) - 2 * (lit > 0 ? 1 : 0) - 11;
+ if (score > bscore) {
+ blen = l;
+ bp = p;
+ blit = 0;
+ bscore = score;
+ }
+ }
+ }
+ if (blen >= 128)
+ break;
+ }
+ }
+ }
+
+ var off = i - bp;
+ var extra = this._level == 2 ? (off >= 1 << 16 ? 1u : 0u) + (off >= 1 << 24 ? 1u : 0u) : 0u;
+ if (off > 0 && bscore > 0 && blen - blit >= this._minMatch + extra) {
+ lit += blit;
+ this.WriteLiteral(i + blit, ref lit);
+ this.WriteMatch(blen - blit, off);
+ } else {
+ blen = 1;
+ ++lit;
+ }
+
+ if (this._isaOffset >= 0)
+ this._i += blen;
+ else {
+ while (blen-- > 0) {
+ i = this._i;
+ if (i + this._minMatchBoth < n) {
+ var ih = ((i * 1234547u) >> 19) & this._bucket;
+ var p = (i << this._checkbits) | (input[i + 3] & mask);
+ if (this._minMatch2 != 0) {
+ ht[this._h2 ^ ih] = p;
+ this._h2 = (((this._h2 * 9) << this._shift2) + (input[i + this._minMatch2 + this._lookahead] + 1u) * 23456789u) & (this._htsize - 1);
+ }
+ ht[this._h1 ^ ih] = p;
+ this._h1 = (((this._h1 * 5) << this._shift1) + (input[i + this._minMatch] + 1u) * 123456791u) & (this._htsize - 1);
+ }
+ ++this._i;
+ }
+ }
+
+ if (lit >= this._maxLiteral)
+ this.WriteLiteral(this._i, ref lit);
+ }
+
+ if (this._i == n) {
+ this.WriteLiteral(n, ref lit);
+ this.FlushBits();
+ }
+ }
+
+ private void WriteLiteral(uint i, ref uint lit) {
+ var input = this._in;
+ if (this._level == 1) {
+ if (lit < 1)
+ return;
+ var ll = Lg(lit);
+ this.PutBits(0, 2);
+ --ll;
+ while (--ll >= 0) {
+ this.PutBits(1, 1);
+ this.PutBits((lit >> ll) & 1, 1);
+ }
+ this.PutBits(0, 1);
+ while (lit != 0)
+ this.PutBits(input[i - lit--], 8);
+ return;
+ }
+ while (lit > 0) {
+ var lit1 = Math.Min(lit, 64u);
+ this.Put((int)lit1 - 1);
+ for (var j = i - lit; j < i - lit + lit1; ++j)
+ this.Put(input[j]);
+ lit -= lit1;
+ }
+ }
+
+ private void WriteMatch(uint len, uint off) {
+ if (this._level == 1) {
+ var ll = Lg(len) - 1;
+ off += (1u << this._rb) - 1;
+ var lo = Lg(off) - 1 - this._rb;
+ this.PutBits((uint)(lo + 8) >> 3, 2);
+ this.PutBits((uint)lo & 7, 3);
+ while (--ll >= 2) {
+ this.PutBits(1, 1);
+ this.PutBits((len >> ll) & 1, 1);
+ }
+ this.PutBits(0, 1);
+ this.PutBits(len & 3, 2);
+ this.PutBits(off, this._rb);
+ this.PutBits(off >> this._rb, lo);
+ return;
+ }
+ --off;
+ var minMatch = this._minMatch;
+ while (len > 0) {
+ var len1 = len > minMatch * 2 + 63 ? minMatch + 63 : len > minMatch + 63 ? len - minMatch : len;
+ if (off < 1 << 16) {
+ this.Put((int)(64 + len1 - minMatch));
+ this.Put((int)(off >> 8));
+ this.Put((int)off);
+ } else if (off < 1 << 24) {
+ this.Put((int)(128 + len1 - minMatch));
+ this.Put((int)(off >> 16));
+ this.Put((int)(off >> 8));
+ this.Put((int)off);
+ } else {
+ this.Put((int)(192 + len1 - minMatch));
+ this.Put((int)(off >> 24));
+ this.Put((int)(off >> 16));
+ this.Put((int)(off >> 8));
+ this.Put((int)off);
+ }
+ len -= len1;
+ }
+ }
+}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqMethods.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqMethods.cs
new file mode 100644
index 000000000..b5593090c
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqMethods.cs
@@ -0,0 +1,827 @@
+using System.Globalization;
+using System.Text;
+
+namespace FileFormat.Zpaq;
+
+///
+/// ZPAQ compression methods: the method-string language (0..5, x...), the
+/// ZPAQL configurations it expands to, and single-block compression with the LZ77 / BWT /
+/// E8E9 preprocessors.
+///
+///
+/// Converted from libzpaq 7.15 (Matt Mahoney, public domain): makeConfig,
+/// compressBlock and e8e9. The generated ZPAQL text is carried over verbatim,
+/// because it is what ends up in every block header and the reference decoder runs it.
+///
+internal static class ZpaqMethods {
+
+ /// Decimal representation of a non-negative number, at least digits.
+ internal static string Itos(long x, int n = 1) {
+ var s = x.ToString(CultureInfo.InvariantCulture);
+ return s.Length >= n ? s : new string('0', n - s.Length) + s;
+ }
+
+ /// floor(log2(x)) + 1: the number of significant bits of .
+ internal static int Lg(int x) => x == 0 ? 0 : 32 - System.Numerics.BitOperations.LeadingZeroCount((uint)x);
+
+ /// Number of set bits of .
+ internal static int NBits(int x) => System.Numerics.BitOperations.PopCount((uint)x);
+
+ ///
+ /// E8E9 transform: scanning backward, adds the block offset to the 32-bit operand of every
+ /// E8/E9 xx xx xx 00/FF (x86 CALL/JMP) so that repeated targets compress better.
+ ///
+ internal static void E8E9(Span buf) {
+ for (var i = buf.Length - 5; i >= 0; --i) {
+ if ((buf[i] & 254) != 0xe8 || ((buf[i + 4] + 1) & 254) != 0)
+ continue;
+ var a = (uint)(buf[i + 1] | buf[i + 2] << 8 | buf[i + 3] << 16) + (uint)i;
+ buf[i + 1] = (byte)a;
+ buf[i + 2] = (byte)(a >> 8);
+ buf[i + 3] = (byte)(a >> 16);
+ }
+ }
+
+ ///
+ /// Expands an explicit method string (0..., x..., s..., i...) into a ZPAQL
+ /// configuration, filling ($1..$9) from its numeric arguments.
+ ///
+ internal static string MakeConfig(string method, int[] args) {
+ var type = method[0];
+ Array.Clear(args, 0, 9);
+ var mi = 1;
+ for (var i = 0; i < 9 && mi < method.Length && (char.IsAsciiDigit(method[mi]) || method[mi] == ',' || method[mi] == '.');) {
+ if (char.IsAsciiDigit(method[mi]))
+ args[i] = args[i] * 10 + method[mi] - '0';
+ else if (++i < 9)
+ args[i] = 0;
+ ++mi;
+ }
+ // "0..." = No compression
+ if (type == '0')
+ return "comp 0 0 0 0 0 hcomp end\n";
+
+ // Generate the postprocessor
+ string hdr = "", pcomp = "";
+ var level = args[1] & 3;
+ var doe8 = args[1] >= 4 && args[1] <= 7;
+
+ // LZ77+Huffman, with or without E8E9
+ if (level == 1) {
+ var rb = args[0] > 4 ? args[0] - 4 : 0;
+ hdr = "comp 9 16 0 $1+20 ";
+ pcomp =
+ "pcomp lazy2 3 ;\n" +
+ " (r1 = state\n" +
+ " r2 = len - match or literal length\n" +
+ " r3 = m - number of offset bits expected\n" +
+ " r4 = ptr to buf\n" +
+ " r5 = r - low bits of offset\n" +
+ " c = bits - input buffer\n" +
+ " d = n - number of bits in c)\n" +
+ "\n" +
+ " a> 255 if\n";
+ if (doe8)
+ pcomp +=
+ " b=0 d=r 4 do (for b=0..d-1, d = end of buf)\n" +
+ " a=b a==d ifnot\n" +
+ " a+= 4 a>= 8 b++\n" +
+ " *b=a a>>= 8 b++\n" +
+ " *b=a b++\n" +
+ " endif\n" +
+ " b=c\n" +
+ " endif\n" +
+ " endif\n" +
+ " a=*b out b++\n" +
+ " forever\n" +
+ " endif\n" +
+ "\n";
+ pcomp +=
+ " (reset state)\n" +
+ " a=0 b=0 c=0 d=0 r=a 1 r=a 2 r=a 3 r=a 4\n" +
+ " halt\n" +
+ " endif\n" +
+ "\n" +
+ " a<<=d a+=c c=a (bits+=a< 0 if (if (bits&3))\n" +
+ " a-- a<<= 3 r=a 3 (m=((bits&3)-1)*8)\n" +
+ " a=c a>>= 2 c=a (bits>>=2)\n" +
+ " b=r 3 a&= 7 a+=b r=a 3 (m+=bits&7)\n" +
+ " a=c a>>= 3 c=a (bits>>=3)\n" +
+ " a=d a-= 5 d=a (n-=5)\n" +
+ " a= 1 r=a 1 (state=1)\n" +
+ " else (literal, discard 00)\n" +
+ " a=c a>>= 2 c=a (bits>>=2)\n" +
+ " d-- d-- (n-=2)\n" +
+ " a= 3 r=a 1 (state=3)\n" +
+ " endif\n" +
+ " endif\n" +
+ "\n" +
+ " (while state==1 && n>=3 (expect match length n*4+ll -> r2))\n" +
+ " do a=r 1 a== 1 if a=d a> 2 if\n" +
+ " a=c a&= 1 a== 1 if (if bits&1)\n" +
+ " a=c a>>= 1 c=a (bits>>=1)\n" +
+ " b=r 2 a=c a&= 1 a+=b a+=b r=a 2 (len+=len+(bits&1))\n" +
+ " a=c a>>= 1 c=a (bits>>=1)\n" +
+ " d-- d-- (n-=2)\n" +
+ " else\n" +
+ " a=c a>>= 1 c=a (bits>>=1)\n" +
+ " a=r 2 a<<= 2 b=a (len<<=2)\n" +
+ " a=c a&= 3 a+=b r=a 2 (len+=bits&3)\n" +
+ " a=c a>>= 2 c=a (bits>>=2)\n" +
+ " d-- d-- d-- (n-=3)\n";
+ if (rb != 0)
+ pcomp += " a= 5 r=a 1 (state=5)\n";
+ else
+ pcomp += " a= 2 r=a 1 (state=2)\n";
+ pcomp +=
+ " endif\n" +
+ " forever endif endif\n" +
+ "\n";
+ if (rb != 0) pcomp += // save r in r5
+ " (if state==5 && n>=8) (expect low bits of offset to put in r5)\n" +
+ " a=r 1 a== 5 if a=d a> " + Itos(rb - 1) + " if\n" +
+ " a=c a&= " + Itos((1 << rb) - 1) + " r=a 5 (save r in r5)\n" +
+ " a=c a>>= " + Itos(rb) + " c=a\n" +
+ " a=d a-= " + Itos(rb) + " d=a\n" +
+ " a= 2 r=a 1 (go to state 2)\n" +
+ " endif endif\n" +
+ "\n";
+ pcomp +=
+ " (if state==2 && n>=m) (expect m offset bits)\n" +
+ " a=r 1 a== 2 if a=r 3 a>d ifnot\n" +
+ " a=c r=a 6 a=d r=a 7 (save c=bits, d=n in r6,r7)\n" +
+ " b=r 3 a= 1 a<<=b d=a (d=1< 0 if d--\n" +
+ " a=*c *b=a c++ b++ (buf[ptr++]-buf[p++])\n";
+ if (!doe8) pcomp += " out\n";
+ pcomp +=
+ " forever endif\n" +
+ " a=b r=a 4\n" +
+ "\n" +
+ " a=r 6 b=r 3 a>>=b c=a (bits>>=m)\n" +
+ " a=r 7 a-=b d=a (n-=m)\n" +
+ " a=0 r=a 1 (state=0)\n" +
+ " endif endif\n" +
+ "\n" +
+ " (while state==3 && n>=2 (expect literal length))\n" +
+ " do a=r 1 a== 3 if a=d a> 1 if\n" +
+ " a=c a&= 1 a== 1 if (if bits&1)\n" +
+ " a=c a>>= 1 c=a (bits>>=1)\n" +
+ " b=r 2 a&= 1 a+=b a+=b r=a 2 (len+=len+(bits&1))\n" +
+ " a=c a>>= 1 c=a (bits>>=1)\n" +
+ " d-- d-- (n-=2)\n" +
+ " else\n" +
+ " a=c a>>= 1 c=a (bits>>=1)\n" +
+ " d-- (--n)\n" +
+ " a= 4 r=a 1 (state=4)\n" +
+ " endif\n" +
+ " forever endif endif\n" +
+ "\n" +
+ " (if state==4 && n>=8 (expect len literals))\n" +
+ " a=r 1 a== 4 if a=d a> 7 if\n" +
+ " b=r 4 a=c *b=a\n";
+ if (!doe8) pcomp += " out\n";
+ pcomp +=
+ " b++ a=b r=a 4 (buf[ptr++]=bits)\n" +
+ " a=c a>>= 8 c=a (bits>>=8)\n" +
+ " a=d a-= 8 d=a (n-=8)\n" +
+ " a=r 2 a-- r=a 2 a== 0 if (if --len<1)\n" +
+ " a=0 r=a 1 (state=0)\n" +
+ " endif\n" +
+ " endif endif\n" +
+ " halt\n" +
+ "end\n";
+ }
+
+ // Byte aligned LZ77, with or without E8E9
+ else if (level == 2) {
+ hdr = "comp 9 16 0 $1+20 ";
+ pcomp =
+ "pcomp lzpre c ;\n" +
+ " (Decode LZ77: d=state, M=output buffer, b=size)\n" +
+ " a> 255 if (at EOF decode e8e9 and output)\n";
+ if (doe8)
+ pcomp +=
+ " d=b b=0 do (for b=0..d-1, d = end of buf)\n" +
+ " a=b a==d ifnot\n" +
+ " a+= 4 a>= 8 b++\n" +
+ " *b=a a>>= 8 b++\n" +
+ " *b=a b++\n" +
+ " endif\n" +
+ " b=c\n" +
+ " endif\n" +
+ " endif\n" +
+ " a=*b out b++\n" +
+ " forever\n" +
+ " endif\n";
+ pcomp +=
+ " b=0 c=0 d=0 a=0 r=a 1 r=a 2 (reset state)\n" +
+ " halt\n" +
+ " endif\n" +
+ "\n" +
+ " (in state d==0, expect a new code)\n" +
+ " (put length in r1 and inital part of offset in r2)\n" +
+ " c=a a=d a== 0 if\n" +
+ " a=c a>>= 6 a++ d=a\n" +
+ " a== 1 if (literal?)\n" +
+ " a+=c r=a 1 a=0 r=a 2\n" +
+ " else (3 to 5 byte match)\n" +
+ " d++ a=c a&= 63 a+= $3 r=a 1 a=0 r=a 2\n" +
+ " endif\n" +
+ " else\n" +
+ " a== 1 if (writing literal)\n" +
+ " a=c *b=a b++\n";
+ if (!doe8) pcomp += " out\n";
+ pcomp +=
+ " a=r 1 a-- a== 0 if d=0 endif r=a 1 (if (--len==0) state=0)\n" +
+ " else\n" +
+ " a> 2 if (reading offset)\n" +
+ " a=r 2 a<<= 8 a|=c r=a 2 d-- (off=off<<8|c, --state)\n" +
+ " else (state==2, write match)\n" +
+ " a=r 2 a<<= 8 a|=c c=a a=b a-=c a-- c=a (c=i-off-1)\n" +
+ " d=r 1 (d=len)\n" +
+ " do (copy and output d=len bytes)\n" +
+ " a=*c *b=a c++ b++\n";
+ if (!doe8) pcomp += " out\n";
+ pcomp +=
+ " d-- a=d a> 0 while\n" +
+ " (d=state=0. off, len don\'t matter)\n" +
+ " endif\n" +
+ " endif\n" +
+ " endif\n" +
+ " halt\n" +
+ "end\n";
+ }
+
+ // BWT with or without E8E9
+ else if (level == 3) { // IBWT
+ hdr = "comp 9 16 $1+20 $1+20 "; // 2^$1 = block size in MB
+ pcomp =
+ "pcomp bwtrle c ;\n" +
+ "\n" +
+ " (read BWT, index into M, size in b)\n" +
+ " a> 255 ifnot\n" +
+ " *b=a b++\n" +
+ "\n" +
+ " (inverse BWT)\n" +
+ " elsel\n" +
+ "\n" +
+ " (index in last 4 bytes, put in c and R1)\n" +
+ " b-- a=*b\n" +
+ " b-- a<<= 8 a+=*b\n" +
+ " b-- a<<= 8 a+=*b\n" +
+ " b-- a<<= 8 a+=*b c=a r=a 1\n" +
+ "\n" +
+ " (save size in R2)\n" +
+ " a=b r=a 2\n" +
+ "\n" +
+ " (count bytes in H[~1..~255, ~0])\n" +
+ " do\n" +
+ " a=b a> 0 if\n" +
+ " b-- a=*b a++ a&= 255 d=a d! *d++\n" +
+ " forever\n" +
+ " endif\n" +
+ "\n" +
+ " (cumulative counts: H[~i=0..255] = count of bytes before i)\n" +
+ " d=0 d! *d= 1 a=0\n" +
+ " do\n" +
+ " a+=*d *d=a d--\n" +
+ " d<>a a! a> 255 a! d<>a until\n" +
+ "\n" +
+ " (build first part of linked list in H[0..idx-1])\n" +
+ " b=0 do\n" +
+ " a=c a>b if\n" +
+ " d=*b d! *d++ d=*d d-- *d=b\n" +
+ " b++ forever\n" +
+ " endif\n" +
+ "\n" +
+ " (rest of list in H[idx+1..n-1])\n" +
+ " b=c b++ c=r 2 do\n" +
+ " a=c a>b if\n" +
+ " d=*b d! *d++ d=*d d-- *d=b\n" +
+ " b++ forever\n" +
+ " endif\n" +
+ "\n";
+ if (args[0] <= 4) { // faster IBWT list traversal limited to 16 MB blocks
+ pcomp +=
+ " (copy M to low 8 bits of H to reduce cache misses in next loop)\n" +
+ " b=0 do\n" +
+ " a=c a>b if\n" +
+ " d=b a=*d a<<= 8 a+=*b *d=a\n" +
+ " b++ forever\n" +
+ " endif\n" +
+ "\n" +
+ " (traverse list and output or copy to M)\n" +
+ " d=r 1 b=0 do\n" +
+ " a=d a== 0 ifnot\n" +
+ " a=*d a>>= 8 d=a\n";
+ if (doe8) pcomp += " *b=*d b++\n";
+ else pcomp += " a=*d out\n";
+ pcomp +=
+ " forever\n" +
+ " endif\n" +
+ "\n";
+ if (doe8) // IBWT+E8E9
+ pcomp +=
+ " (e8e9 transform to out)\n" +
+ " d=b b=0 do (for b=0..d-1, d = end of buf)\n" +
+ " a=b a==d ifnot\n" +
+ " a+= 4 a>= 8 b++\n" +
+ " *b=a a>>= 8 b++\n" +
+ " *b=a b++\n" +
+ " endif\n" +
+ " b=c\n" +
+ " endif\n" +
+ " endif\n" +
+ " a=*b out b++\n" +
+ " forever\n" +
+ " endif\n";
+ pcomp +=
+ " endif\n" +
+ " halt\n" +
+ "end\n";
+ } else { // slower IBWT list traversal for all sized blocks
+ if (doe8) { // E8E9 after IBWT
+ pcomp +=
+ " (R2 = output size without EOS)\n" +
+ " a=r 2 a-- r=a 2\n" +
+ "\n" +
+ " (traverse list (d = IBWT pointer) and output inverse e8e9)\n" +
+ " (C = offset = 0..R2-1)\n" +
+ " (R4 = last 4 bytes shifted in from MSB end)\n" +
+ " (R5 = temp pending output byte)\n" +
+ " c=0 d=r 1 do\n" +
+ " a=d a== 0 ifnot\n" +
+ " d=*d\n" +
+ "\n" +
+ " (store byte in R4 and shift out to R5)\n" +
+ " b=d a=*b a<<= 24 b=a\n" +
+ " a=r 4 r=a 5 a>>= 8 a|=b r=a 4\n" +
+ "\n" +
+ " (if E8|E9 xx xx xx 00|FF in R4:R5 then subtract c from x)\n" +
+ " a=c a> 3 if\n" +
+ " a=r 5 a&= 254 a== 232 if\n" +
+ " a=r 4 a>>= 24 b=a a++ a&= 254 a< 2 if\n" +
+ " a=r 4 a-=c a+= 4 a<<= 8 a>>= 8 \n" +
+ " b<>a a<<= 24 a+=b r=a 4\n" +
+ " endif\n" +
+ " endif\n" +
+ " endif\n" +
+ "\n" +
+ " (output buffered byte)\n" +
+ " a=c a> 3 if a=r 5 out endif c++\n" +
+ "\n" +
+ " forever\n" +
+ " endif\n" +
+ "\n" +
+ " (output up to 4 pending bytes in R4)\n" +
+ " b=r 4\n" +
+ " a=c a> 3 a=b if out endif a>>= 8 b=a\n" +
+ " a=c a> 2 a=b if out endif a>>= 8 b=a\n" +
+ " a=c a> 1 a=b if out endif a>>= 8 b=a\n" +
+ " a=c a> 0 a=b if out endif\n" +
+ "\n" +
+ " endif\n" +
+ " halt\n" +
+ "end\n";
+ } else {
+ pcomp +=
+ " (traverse list and output)\n" +
+ " d=r 1 do\n" +
+ " a=d a== 0 ifnot\n" +
+ " d=*d\n" +
+ " b=d a=*b out\n" +
+ " forever\n" +
+ " endif\n" +
+ " endif\n" +
+ " halt\n" +
+ "end\n";
+ }
+ }
+ }
+
+ // E8E9 or no preprocessing
+ else if (level == 0) {
+ hdr = "comp 9 16 0 0 ";
+ if (doe8) { // E8E9?
+ pcomp =
+ "pcomp e8e9 d ;\n" +
+ " a> 255 if\n" +
+ " a=c a> 4 if\n" +
+ " c= 4\n" +
+ " else\n" +
+ " a! a+= 5 a<<= 3 d=a a=b a>>=d b=a\n" +
+ " endif\n" +
+ " do a=c a> 0 if\n" +
+ " a=b out a>>= 8 b=a c--\n" +
+ " forever endif\n" +
+ " else\n" +
+ " *b=b a<<= 24 d=a a=b a>>= 8 a+=d b=a c++\n" +
+ " a=c a> 4 if\n" +
+ " a=*b out\n" +
+ " a&= 254 a== 232 if\n" +
+ " a=b a>>= 24 a++ a&= 254 a== 0 if\n" +
+ " a=b a>>= 24 a<<= 24 d=a\n" +
+ " a=b a-=c a+= 5\n" +
+ " a<<= 8 a>>= 8 a|=d b=a\n" +
+ " endif\n" +
+ " endif\n" +
+ " endif\n" +
+ " endif\n" +
+ " halt\n" +
+ "end\n";
+ } else
+ pcomp = "end\n";
+ } else
+ ZpaqError.Throw("Unsupported method");
+
+ // Build context model (comp, hcomp) assuming:
+ // H[0..254] = contexts
+ // H[255..511] = location of last byte i-255
+ // M = last 64K bytes, filling backward
+ // C = pointer to most recent byte
+ // R1 = level 2 lz77 1+bytes expected until next code, 0=init
+ // R2 = level 2 lz77 first byte of code
+ var ncomp = 0; // number of components
+ var membits = args[0] + 20;
+ var sb = 5; // bits in last context
+ var comp = "";
+ string hcomp = "hcomp\n" +
+ "c-- *c=a a+= 255 d=a *d=c\n";
+ if (level == 2) { // put level 2 lz77 parse state in R1, R2
+ hcomp +=
+ " (decode lz77 into M. Codes:\n" +
+ " 00xxxxxx = literal length xxxxxx+1\n" +
+ " xx......, xx > 0 = match with xx offset bytes to follow)\n" +
+ "\n" +
+ " a=r 1 a== 0 if (init)\n" +
+ " a= " + Itos(111 + (doe8 ? 57 : 0)) + " (skip post code)\n" +
+ " else a== 1 if (new code?)\n" +
+ " a=*c r=a 2 (save code in R2)\n" +
+ " a> 63 if a>>= 6 a++ a++ (match)\n" +
+ " else a++ a++ endif (literal)\n" +
+ " else (read rest of code)\n" +
+ " a--\n" +
+ " endif endif\n" +
+ " r=a 1 (R1 = 1+expected bytes to next code)\n";
+ }
+
+ // Generate the context model
+
+ while (mi < method.Length && ncomp < 254) {
+
+ // parse command C[N1[,N2]...] into v = {C, N1, N2...}
+ var v = new List { method[mi++] };
+ if (mi < method.Length && char.IsAsciiDigit(method[mi])) {
+ v.Add(method[mi++] - '0');
+ while (mi < method.Length && (char.IsAsciiDigit(method[mi]) || method[mi] == ',' || method[mi] == '.')) {
+ if (char.IsAsciiDigit(method[mi]))
+ v[^1] = v[^1] * 10 + method[mi++] - '0';
+ else {
+ v.Add(0);
+ ++mi;
+ }
+ }
+ }
+
+ // c: context model
+ // N1%1000: 0=ICM 1..256=CM limit N1-1
+ // N1/1000: number of times to halve memory
+ // N2: 1..255=offset mod N2. 1000..1255=distance to N2-1000
+ // N3...: 0..255=byte mask + 256=lz77 state. 1000+=run of N3-1000 zeros.
+ if (v[0] == 'c') {
+ while (v.Count < 3) v.Add(0);
+ comp += Itos(ncomp) + " ";
+ sb = 11; // count context bits
+ if (v[2] < 256) sb += Lg(v[2]);
+ else sb += 6;
+ for (var i = 3; i < v.Count; ++i)
+ if (v[i] < 512) sb += NBits(v[i]) * 3 / 4;
+ if (sb > membits) sb = membits;
+ if (v[1] % 1000 == 0) comp += "icm " + Itos(sb - 6 - v[1] / 1000) + "\n";
+ else comp += "cm " + Itos(sb - 2 - v[1] / 1000) + " " + Itos(v[1] % 1000 - 1) + "\n";
+
+ // special contexts
+ hcomp += "d= " + Itos(ncomp) + " *d=0\n";
+ if (v[2] > 1 && v[2] <= 255) { // periodic context
+ if (Lg(v[2]) != Lg(v[2] - 1))
+ hcomp += "a=c a&= " + Itos(v[2] - 1) + " hashd\n";
+ else
+ hcomp += "a=c a%= " + Itos(v[2]) + " hashd\n";
+ } else if (v[2] >= 1000 && v[2] <= 1255) // distance context
+ hcomp += "a= 255 a+= " + Itos(v[2] - 1000) +
+ " d=a a=*d a-=c a> 255 if a= 255 endif d= " +
+ Itos(ncomp) + " hashd\n";
+
+ // Masked context
+ for (var i = 3; i < v.Count; ++i) {
+ if (i == 3) hcomp += "b=c ";
+ if (v[i] == 255)
+ hcomp += "a=*b hashd\n"; // ordinary byte
+ else if (v[i] > 0 && v[i] < 255)
+ hcomp += "a=*b a&= " + Itos(v[i]) + " hashd\n"; // masked byte
+ else if (v[i] >= 256 && v[i] < 512) { // lz77 state or masked literal byte
+ hcomp +=
+ "a=r 1 a> 1 if\n" + // expect literal or offset
+ " a=r 2 a< 64 if\n" + // expect literal
+ " a=*b ";
+ if (v[i] < 511) hcomp += "a&= " + Itos(v[i] - 256);
+ hcomp += " hashd\n" +
+ " else\n" + // expect match offset byte
+ " a>>= 6 hashd a=r 1 hashd\n" +
+ " endif\n" +
+ "else\n" + // expect new code
+ " a= 255 hashd a=r 2 hashd\n" +
+ "endif\n";
+ } else if (v[i] >= 1256) // skip v[i]-1000 bytes
+ hcomp += "a= " + Itos(((v[i] - 1000) >> 8) & 255) + " a<<= 8 a+= "
+ + Itos((v[i] - 1000) & 255) +
+ " a+=b b=a\n";
+ else if (v[i] > 1000)
+ hcomp += "a= " + Itos(v[i] - 1000) + " a+=b b=a\n";
+ if (v[i] < 512 && i < v.Count - 1)
+ hcomp += "b++ ";
+ }
+ ++ncomp;
+ }
+
+ // m,8,24: MIX, size, rate
+ // t,8,24: MIX2, size, rate
+ // s,8,32,255: SSE, size, start, limit
+ if ("mts".Contains((char)v[0]) && ncomp > (v[0] == 't' ? 1 : 0)) {
+ if (v.Count <= 1) v.Add(8);
+ if (v.Count <= 2) v.Add(24 + (v[0] == 's' ? 8 : 0));
+ if (v[0] == 's' && v.Count <= 3) v.Add(255);
+ comp += Itos(ncomp);
+ sb = 5 + v[1] * 3 / 4;
+ if (v[0] == 'm')
+ comp += " mix " + Itos(v[1]) + " 0 " + Itos(ncomp) + " " + Itos(v[2]) + " 255\n";
+ else if (v[0] == 't')
+ comp += " mix2 " + Itos(v[1]) + " " + Itos(ncomp - 1) + " " + Itos(ncomp - 2)
+ + " " + Itos(v[2]) + " 255\n";
+ else // s
+ comp += " sse " + Itos(v[1]) + " " + Itos(ncomp - 1) + " " + Itos(v[2]) + " "
+ + Itos(v[3]) + "\n";
+ if (v[1] > 8) {
+ hcomp += "d= " + Itos(ncomp) + " *d=0 b=c a=0\n";
+ for (; v[1] >= 16; v[1] -= 8) {
+ hcomp += "a<<= 8 a+=*b";
+ if (v[1] > 16) hcomp += " b++";
+ hcomp += "\n";
+ }
+ if (v[1] > 8)
+ hcomp += "a<<= 8 a+=*b a>>= " + Itos(16 - v[1]) + "\n";
+ hcomp += "a<<= 8 *d=a\n";
+ }
+ ++ncomp;
+ }
+
+ // i: ISSE chain with order increasing by N1,N2...
+ if (v[0] == 'i' && ncomp > 0) {
+ hcomp += "d= " + Itos(ncomp - 1) + " b=c a=*d d++\n";
+ for (var i = 1; i < v.Count && ncomp < 254; ++i) {
+ for (int j = 0; j < v[i] % 10; ++j) {
+ hcomp += "hash ";
+ if (i < v.Count - 1 || j < v[i] % 10 - 1) hcomp += "b++ ";
+ sb += 6;
+ }
+ hcomp += "*d=a";
+ if (i < v.Count - 1) hcomp += " d++";
+ hcomp += "\n";
+ if (sb > membits) sb = membits;
+ comp += Itos(ncomp) + " isse " + Itos(sb - 6 - v[i] / 10) + " " + Itos(ncomp - 1) + "\n";
+ ++ncomp;
+ }
+ }
+
+ // a24,0,0: MATCH. N1=hash multiplier. N2,N3=halve buf, table.
+ if (v[0] == 'a') {
+ if (v.Count <= 1) v.Add(24);
+ while (v.Count < 4) v.Add(0);
+ comp += Itos(ncomp) + " match " + Itos(membits - v[3] - 2) + " "
+ + Itos(membits - v[2]) + "\n";
+ hcomp += "d= " + Itos(ncomp) + " a=*d a*= " + Itos(v[1])
+ + " a+=*c a++ *d=a\n";
+ sb = 5 + (membits - v[2]) * 3 / 4;
+ ++ncomp;
+ }
+
+ // w1,65,26,223,20,0: ICM-ISSE chain of length N1 with word contexts,
+ // where a word is a sequence of c such that c&N4 is in N2..N2+N3-1.
+ // Word is hashed by: hash := hash*N5+c+1
+ // Decrease memory by 2^-N6.
+ if (v[0] == 'w') {
+ if (v.Count <= 1) v.Add(1);
+ if (v.Count <= 2) v.Add(65);
+ if (v.Count <= 3) v.Add(26);
+ if (v.Count <= 4) v.Add(223);
+ if (v.Count <= 5) v.Add(20);
+ if (v.Count <= 6) v.Add(0);
+ comp += Itos(ncomp) + " icm " + Itos(membits - 6 - v[6]) + "\n";
+ for (int i = 1; i < v[1]; ++i)
+ comp += Itos(ncomp + i) + " isse " + Itos(membits - 6 - v[6]) + " "
+ + Itos(ncomp + i - 1) + "\n";
+ hcomp += "a=*c a&= " + Itos(v[4]) + " a-= " + Itos(v[2]) + " a&= 255 a< "
+ + Itos(v[3]) + " if\n";
+ for (int i = 0; i < v[1]; ++i) {
+ if (i == 0) hcomp += " d= " + Itos(ncomp);
+ else hcomp += " d++";
+ hcomp += " a=*d a*= " + Itos(v[5]) + " a+=*c a++ *d=a\n";
+ }
+ hcomp += "else\n";
+ for (int i = v[1] - 1; i > 0; --i)
+ hcomp += " d= " + Itos(ncomp + i - 1) + " a=*d d++ *d=a\n";
+ hcomp += " d= " + Itos(ncomp) + " *d=0\n" +
+ "endif\n";
+ ncomp += v[1] - 1;
+ sb = membits - v[6];
+ ++ncomp;
+ }
+ }
+ return hdr + Itos(ncomp) + "\n" + comp + hcomp + "halt\n" + pcomp;
+ }
+
+ ///
+ /// Compresses into a single block of one segment, choosing the
+ /// algorithm from exactly as libzpaq's compressBlock.
+ ///
+ /// The block's data; E8E9 may transform it in place.
+ /// Receives the block, preceded by the locator tag.
+ ///
+ /// "LB,R,t" (level 0..5, block size, redundancy 0..255, type: 1 = text, 2 = x86) or an
+ /// explicit x/0 method.
+ ///
+ /// Segment filename, or .
+ /// Appended to the decimal size in the segment comment, or .
+ /// Whether the segment trailer carries the SHA-1 of the data.
+ internal static void CompressBlock(ZpaqBuffer input, IZpaqWriter output, string methodString, string? filename = null, byte[]? comment = null, bool storeSha1 = true) {
+ var method = methodString;
+ var n = input.Size;
+ var arg0 = Math.Max(Lg(n + 4095) - 20, 0);
+
+ var type = 0u;
+ if (char.IsAsciiDigit(method[0])) {
+ var commas = 0;
+ Span arg = stackalloc int[4];
+ arg.Clear();
+ for (var i = 1; i < method.Length && commas < 4; ++i) {
+ if (method[i] == ',' || method[i] == '.')
+ ++commas;
+ else if (char.IsAsciiDigit(method[i]))
+ arg[commas] = arg[commas] * 10 + method[i] - '0';
+ }
+ type = commas == 0 ? 512u : (uint)(arg[1] * 4 + arg[2]);
+ }
+
+ byte[]? sha1 = storeSha1 ? Compression.Core.Checksums.Sha1.Compute(input.Span) : null;
+
+ if (char.IsAsciiDigit(method[0])) {
+ var level = method[0] - '0';
+ var doe8 = (int)(type & 2) * 2;
+ method = "x" + Itos(arg0);
+ var htsz = "," + Itos(19 + arg0 + (arg0 <= 6 ? 1 : 0));
+ var sasz = "," + Itos(21 + arg0);
+
+ if (level == 0)
+ method = "0" + Itos(arg0) + ",0";
+ else if (level == 1) {
+ if (type < 40)
+ method += ",0";
+ else {
+ method += "," + Itos(1 + doe8) + ",";
+ if (type < 80) method += "4,0,1,15";
+ else if (type < 128) method += "4,0,2,16";
+ else if (type < 256) method += "4,0,2" + htsz;
+ else if (type < 960) method += "5,0,3" + htsz;
+ else method += "6,0,3" + htsz;
+ }
+ } else if (level == 2) {
+ if (type < 32)
+ method += ",0";
+ else {
+ method += "," + Itos(1 + doe8) + ",";
+ if (type < 64) method += "4,0,3" + htsz;
+ else method += "4,0,7" + sasz + ",1";
+ }
+ } else if (level == 3) {
+ if (type < 20)
+ method += ",0";
+ else if (type < 48)
+ method += "," + Itos(1 + doe8) + ",4,0,3" + htsz;
+ else if (type >= 640 || (type & 1) != 0)
+ method += "," + Itos(3 + doe8) + "ci1";
+ else
+ method += "," + Itos(2 + doe8) + ",12,0,7" + sasz + ",1c0,0,511i2";
+ } else if (level == 4) {
+ if (type < 12)
+ method += ",0";
+ else if (type < 24)
+ method += "," + Itos(1 + doe8) + ",4,0,3" + htsz;
+ else if (type < 48)
+ method += "," + Itos(2 + doe8) + ",5,0,7" + sasz + "1c0,0,511";
+ else if (type < 900) {
+ method += "," + Itos(doe8) + "ci1,1,1,1,2a";
+ if ((type & 1) != 0)
+ method += "w";
+ method += "m";
+ } else
+ method += "," + Itos(3 + doe8) + "ci1";
+ } else {
+ method += "," + Itos(doe8);
+ method += (type & 1) != 0 ? "w2c0,1010,255i1" : "w1i1";
+ method += "c256ci1,1,1,1,1,1,2a";
+
+ const int nr = 1 << 12;
+ var pt = new int[256];
+ var r = new int[nr];
+ var p = input.Span;
+ if (level > 0) {
+ for (var i = 0; i < n; ++i) {
+ var k = i - pt[p[i]];
+ if (k is > 0 and < nr)
+ ++r[k];
+ pt[p[i]] = i;
+ }
+ }
+
+ var n1 = n - r[1] - r[2] - r[3];
+ for (var i = 0; i < 2; ++i) {
+ var period = 0;
+ var score = 0.0;
+ var t = 0;
+ for (var j = 5; j < nr && t < n1; ++j) {
+ var s = r[j] / (256.0 + n1 - t);
+ if (s > score) {
+ score = s;
+ period = j;
+ }
+ t += r[j];
+ }
+ if (period > 4 && score > 0.1) {
+ method += "c0,0," + Itos(999 + period) + ",255i1";
+ if (period <= 255)
+ method += "c0," + Itos(period) + "i1";
+ n1 -= r[period];
+ r[period] = 0;
+ } else
+ break;
+ }
+ method += "c0,2,0,255i1c0,3,0,0,255i1c0,4,0,0,0,255i1mm16ts19t0";
+ }
+ }
+
+ var args = new int[9];
+ var config = MakeConfig(method, args);
+ var co = new ZpaqCompressor(output);
+ co.WriteTag();
+ co.StartBlock(config, args);
+ var cs = Encoding.ASCII.GetBytes(Itos(n));
+ if (comment != null)
+ cs = [.. cs, (byte)' ', .. comment];
+ co.StartSegment(filename, cs);
+ if (args[1] is >= 1 and <= 7 && args[1] != 4)
+ co.Compress(new ZpaqLzBuffer(input, args));
+ else {
+ if (args[1] is >= 4 and <= 7)
+ E8E9(input.Span);
+ co.Compress(input);
+ }
+ co.EndSegment(sha1);
+ co.EndBlock();
+ }
+}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqPredictor.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqPredictor.cs
new file mode 100644
index 000000000..01ab4be45
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqPredictor.cs
@@ -0,0 +1,398 @@
+namespace FileFormat.Zpaq;
+
+///
+/// Bit predictor of a ZPAQ block: runs the components declared in COMP over the contexts
+/// that HCOMP computes, and learns from every coded bit.
+///
+///
+/// Converted from libzpaq 7.15 (Matt Mahoney, public domain), class Predictor,
+/// interpreted path (predict0/update0). Integer behaviour, including the
+/// wrap-around of the 32-bit products, follows the reference exactly because the coded
+/// stream depends on every bit of every prediction.
+///
+internal sealed class ZpaqPredictor {
+ private readonly ZpaqlMachine _z;
+ private readonly Component[] _comp = new Component[256];
+ private readonly int[] _p = new int[256];
+ private readonly uint[] _h = new uint[256];
+ private int _c8 = 1;
+ private int _hmap4 = 1;
+
+ private static readonly ushort[] SquashTable = BuildSquash();
+ private static readonly short[] StretchTable = BuildStretch();
+
+ /// Per-component model state, the reference's Component.
+ private sealed class Component {
+ public uint Limit;
+ public uint Cxt;
+ public uint A, B, C;
+ public uint[] Cm = [];
+ public byte[] Ht = [];
+ public ushort[] A16 = [];
+ }
+
+ /// Creates a predictor driven by the HCOMP machine .
+ public ZpaqPredictor(ZpaqlMachine z) {
+ this._z = z;
+ for (var i = 0; i < this._comp.Length; ++i)
+ this._comp[i] = new();
+ }
+
+ /// when the block declares at least one component.
+ public bool IsModeled => this._z.Header[6] != 0;
+
+ /// Builds the model described by the block header.
+ public void Init() {
+ this._z.InitHcomp();
+ Array.Clear(this._h);
+ Array.Clear(this._p);
+ this._c8 = 1;
+ this._hmap4 = 1;
+ for (var i = 0; i < 256; ++i)
+ this._comp[i] = new();
+
+ var header = this._z.Header;
+ int n = header[6];
+ var cp = 7;
+ for (var i = 0; i < n; ++i) {
+ var cr = this._comp[i];
+ switch (header[cp]) {
+ case ZpaqlMachine.Cons:
+ this._p[i] = (header[cp + 1] - 128) * 4;
+ break;
+ case ZpaqlMachine.Cm:
+ if (header[cp + 1] > 32)
+ ZpaqError.Throw("max size for CM is 32");
+ cr.Cm = ZpaqlMachine.AllocatePow2(header[cp + 1]);
+ cr.Limit = (uint)(header[cp + 2] * 4);
+ Array.Fill(cr.Cm, 0x80000000u);
+ break;
+ case ZpaqlMachine.Icm:
+ if (header[cp + 1] > 26)
+ ZpaqError.Throw("max size for ICM is 26");
+ cr.Limit = 1023;
+ cr.Cm = new uint[256];
+ cr.Ht = ZpaqlMachine.AllocatePow2(header[cp + 1], 6);
+ for (var j = 0; j < cr.Cm.Length; ++j)
+ cr.Cm[j] = (uint)CmInit(j);
+ break;
+ case ZpaqlMachine.Match:
+ if (header[cp + 1] > 32 || header[cp + 2] > 32)
+ ZpaqError.Throw("max size for MATCH is 32 32");
+ cr.Cm = ZpaqlMachine.AllocatePow2(header[cp + 1]);
+ cr.Ht = ZpaqlMachine.AllocatePow2(header[cp + 2]);
+ cr.Ht[0] = 1;
+ break;
+ case ZpaqlMachine.Avg:
+ if (header[cp + 1] >= i)
+ ZpaqError.Throw("AVG j >= i");
+ if (header[cp + 2] >= i)
+ ZpaqError.Throw("AVG k >= i");
+ break;
+ case ZpaqlMachine.Mix2:
+ if (header[cp + 1] > 32)
+ ZpaqError.Throw("max size for MIX2 is 32");
+ if (header[cp + 3] >= i)
+ ZpaqError.Throw("MIX2 k >= i");
+ if (header[cp + 2] >= i)
+ ZpaqError.Throw("MIX2 j >= i");
+ cr.C = 1u << header[cp + 1];
+ cr.A16 = ZpaqlMachine.AllocatePow2(header[cp + 1]);
+ Array.Fill(cr.A16, (ushort)32768);
+ break;
+ case ZpaqlMachine.Mix: {
+ if (header[cp + 1] > 32)
+ ZpaqError.Throw("max size for MIX is 32");
+ if (header[cp + 2] >= i)
+ ZpaqError.Throw("MIX j >= i");
+ if (header[cp + 3] < 1 || header[cp + 3] > i - header[cp + 2])
+ ZpaqError.Throw("MIX m not in 1..i-j");
+ int m = header[cp + 3];
+ cr.C = 1u << header[cp + 1];
+ var size = (1L << header[cp + 1]) * m;
+ if (size > Array.MaxLength)
+ throw new NotSupportedException("ZPAQ: MIX weight table exceeds what this runtime can allocate.");
+ cr.Cm = new uint[size];
+ Array.Fill(cr.Cm, (uint)(65536 / m));
+ break;
+ }
+ case ZpaqlMachine.Isse:
+ if (header[cp + 1] > 32)
+ ZpaqError.Throw("max size for ISSE is 32");
+ if (header[cp + 2] >= i)
+ ZpaqError.Throw("ISSE j >= i");
+ cr.Ht = ZpaqlMachine.AllocatePow2(header[cp + 1], 6);
+ cr.Cm = new uint[512];
+ for (var j = 0; j < 256; ++j) {
+ cr.Cm[j * 2] = 1 << 15;
+ cr.Cm[j * 2 + 1] = (uint)Clamp512K(Stretch(CmInit(j) >> 8) * 1024);
+ }
+ break;
+ case ZpaqlMachine.Sse:
+ if (header[cp + 1] > 32)
+ ZpaqError.Throw("max size for SSE is 32");
+ if (header[cp + 2] >= i)
+ ZpaqError.Throw("SSE j >= i");
+ if (header[cp + 3] > header[cp + 4] * 4)
+ ZpaqError.Throw("SSE start > limit*4");
+ cr.Cm = ZpaqlMachine.AllocatePow2(header[cp + 1], 5);
+ cr.Limit = (uint)(header[cp + 4] * 4);
+ for (var j = 0; j < cr.Cm.Length; ++j)
+ cr.Cm[j] = (uint)(Squash((j & 31) * 64 - 992) << 17 | header[cp + 3]);
+ break;
+ default:
+ ZpaqError.Throw("unknown component type");
+ break;
+ }
+ cp += ZpaqlMachine.ComponentSize[header[cp]];
+ }
+ }
+
+ /// Probability (0..32767) that the next bit is a 1.
+ public int Predict() {
+ var header = this._z.Header;
+ var p = this._p;
+ int n = header[6];
+ var cp = 7;
+ for (var i = 0; i < n; ++i) {
+ var cr = this._comp[i];
+ switch (header[cp]) {
+ case ZpaqlMachine.Cons:
+ break;
+ case ZpaqlMachine.Cm:
+ cr.Cxt = this._h[i] ^ (uint)this._hmap4;
+ p[i] = Stretch((int)(cr.Cm[cr.Cxt & (uint)(cr.Cm.Length - 1)] >> 17));
+ break;
+ case ZpaqlMachine.Icm:
+ if (this._c8 == 1 || (this._c8 & 0xf0) == 16)
+ cr.C = Find(cr.Ht, header[cp + 1] + 2, this._h[i] + 16 * (uint)this._c8);
+ cr.Cxt = cr.Ht[cr.C + (this._hmap4 & 15)];
+ p[i] = Stretch((int)(cr.Cm[cr.Cxt & (uint)(cr.Cm.Length - 1)] >> 8));
+ break;
+ case ZpaqlMachine.Match:
+ if (cr.A == 0)
+ p[i] = 0;
+ else {
+ cr.C = (uint)(cr.Ht[(cr.Limit - cr.B) & (uint)(cr.Ht.Length - 1)] >> (int)(7 - cr.Cxt)) & 1;
+ p[i] = Stretch(ZpaqTables.MatchDivisor[cr.A] * (cr.C == 1 ? -1 : 1) & 32767);
+ }
+ break;
+ case ZpaqlMachine.Avg:
+ p[i] = (p[header[cp + 1]] * header[cp + 3] + p[header[cp + 2]] * (256 - header[cp + 3])) >> 8;
+ break;
+ case ZpaqlMachine.Mix2: {
+ cr.Cxt = (this._h[i] + (uint)(this._c8 & header[cp + 5])) & (cr.C - 1);
+ int w = cr.A16[cr.Cxt];
+ p[i] = (w * p[header[cp + 2]] + (65536 - w) * p[header[cp + 3]]) >> 16;
+ break;
+ }
+ case ZpaqlMachine.Mix: {
+ int m = header[cp + 3];
+ cr.Cxt = this._h[i] + (uint)(this._c8 & header[cp + 5]);
+ cr.Cxt = (cr.Cxt & (cr.C - 1)) * (uint)m;
+ var wt = cr.Cm.AsSpan((int)cr.Cxt, m);
+ var sum = 0;
+ int j0 = header[cp + 2];
+ for (var j = 0; j < m; ++j)
+ sum += ((int)wt[j] >> 8) * p[j0 + j];
+ p[i] = Clamp2K(sum >> 8);
+ break;
+ }
+ case ZpaqlMachine.Isse: {
+ if (this._c8 == 1 || (this._c8 & 0xf0) == 16)
+ cr.C = Find(cr.Ht, header[cp + 1] + 2, this._h[i] + 16 * (uint)this._c8);
+ cr.Cxt = cr.Ht[cr.C + (this._hmap4 & 15)];
+ var w0 = (int)cr.Cm[cr.Cxt * 2];
+ var w1 = (int)cr.Cm[cr.Cxt * 2 + 1];
+ p[i] = Clamp2K((w0 * p[header[cp + 2]] + w1 * 64) >> 16);
+ break;
+ }
+ case ZpaqlMachine.Sse: {
+ cr.Cxt = (this._h[i] + (uint)this._c8) * 32;
+ var pq = p[header[cp + 2]] + 992;
+ if (pq < 0)
+ pq = 0;
+ if (pq > 1983)
+ pq = 1983;
+ var wt = pq & 63;
+ pq >>= 6;
+ cr.Cxt += (uint)pq;
+ var mask = (uint)(cr.Cm.Length - 1);
+ p[i] = Stretch((int)(((cr.Cm[cr.Cxt & mask] >> 10) * (uint)(64 - wt) + (cr.Cm[(cr.Cxt + 1) & mask] >> 10) * (uint)wt) >> 13));
+ cr.Cxt += (uint)(wt >> 5);
+ break;
+ }
+ default:
+ ZpaqError.Throw("component predict not implemented");
+ break;
+ }
+ cp += ZpaqlMachine.ComponentSize[header[cp]];
+ }
+ return Squash(p[n - 1]);
+ }
+
+ /// Trains every component on the coded bit and advances the contexts.
+ public void Update(int y) {
+ var header = this._z.Header;
+ var p = this._p;
+ int n = header[6];
+ var cp = 7;
+ for (var i = 0; i < n; ++i) {
+ var cr = this._comp[i];
+ switch (header[cp]) {
+ case ZpaqlMachine.Cons:
+ break;
+ case ZpaqlMachine.Cm:
+ Train(cr, y);
+ break;
+ case ZpaqlMachine.Icm: {
+ var idx = cr.C + (this._hmap4 & 15);
+ cr.Ht[idx] = (byte)Next(cr.Ht[idx], y);
+ ref var pn = ref cr.Cm[cr.Cxt & (uint)(cr.Cm.Length - 1)];
+ pn += (uint)((int)((uint)(y * 32767) - (pn >> 8)) >> 2);
+ break;
+ }
+ case ZpaqlMachine.Match: {
+ var htMask = (uint)(cr.Ht.Length - 1);
+ if ((int)cr.C != y)
+ cr.A = 0;
+ ref var cur = ref cr.Ht[cr.Limit & htMask];
+ cur = (byte)(cur + cur + y);
+ if (++cr.Cxt == 8) {
+ cr.Cxt = 0;
+ ++cr.Limit;
+ cr.Limit &= (uint)((1L << header[cp + 2]) - 1);
+ var cmMask = (uint)(cr.Cm.Length - 1);
+ if (cr.A == 0) {
+ cr.B = cr.Limit - cr.Cm[this._h[i] & cmMask];
+ if ((cr.B & htMask) != 0)
+ while (cr.A < 255 && cr.Ht[(cr.Limit - cr.A - 1) & htMask] == cr.Ht[(cr.Limit - cr.A - cr.B - 1) & htMask])
+ ++cr.A;
+ } else if (cr.A < 255)
+ ++cr.A;
+ cr.Cm[this._h[i] & cmMask] = cr.Limit;
+ }
+ break;
+ }
+ case ZpaqlMachine.Avg:
+ break;
+ case ZpaqlMachine.Mix2: {
+ var err = (y * 32767 - Squash(p[i])) * header[cp + 4] >> 5;
+ int w = cr.A16[cr.Cxt];
+ w += (err * (p[header[cp + 2]] - p[header[cp + 3]]) + (1 << 12)) >> 13;
+ if (w < 0)
+ w = 0;
+ if (w > 65535)
+ w = 65535;
+ cr.A16[cr.Cxt] = (ushort)w;
+ break;
+ }
+ case ZpaqlMachine.Mix: {
+ int m = header[cp + 3];
+ var err = (y * 32767 - Squash(p[i])) * header[cp + 4] >> 4;
+ var wt = cr.Cm.AsSpan((int)cr.Cxt, m);
+ int j0 = header[cp + 2];
+ for (var j = 0; j < m; ++j)
+ wt[j] = (uint)Clamp512K((int)wt[j] + ((err * p[j0 + j] + (1 << 12)) >> 13));
+ break;
+ }
+ case ZpaqlMachine.Isse: {
+ var err = y * 32767 - Squash(p[i]);
+ ref var w0 = ref cr.Cm[cr.Cxt * 2];
+ ref var w1 = ref cr.Cm[cr.Cxt * 2 + 1];
+ w0 = (uint)Clamp512K((int)w0 + ((err * p[header[cp + 2]] + (1 << 12)) >> 13));
+ w1 = (uint)Clamp512K((int)w1 + ((err + 16) >> 5));
+ cr.Ht[cr.C + (this._hmap4 & 15)] = (byte)Next((int)cr.Cxt, y);
+ break;
+ }
+ case ZpaqlMachine.Sse:
+ Train(cr, y);
+ break;
+ }
+ cp += ZpaqlMachine.ComponentSize[header[cp]];
+ }
+
+ this._c8 += this._c8 + y;
+ if (this._c8 >= 256) {
+ this._z.Run((uint)(this._c8 - 256));
+ this._hmap4 = 1;
+ this._c8 = 1;
+ for (var i = 0; i < n; ++i)
+ this._h[i] = this._z.H(i);
+ } else if (this._c8 is >= 16 and < 32)
+ this._hmap4 = (this._hmap4 & 0xf) << 5 | y << 4 | 1;
+ else
+ this._hmap4 = (this._hmap4 & 0x1f0) | (((this._hmap4 & 0xf) * 2 + y) & 0xf);
+ }
+
+ /// Squash: x (-2048..2047) to probability 32768/(1+exp(-x/64)).
+ internal static int Squash(int x) => SquashTable[x + 2048];
+
+ /// Stretch: probability (0..32767) to ln(x/(32768-x))*64, the inverse of squash.
+ internal static int Stretch(int x) => StretchTable[x];
+
+ private static int Clamp2K(int x) => x < -2048 ? -2048 : x > 2047 ? 2047 : x;
+
+ private static int Clamp512K(int x) => x < -(1 << 19) ? -(1 << 19) : x >= 1 << 19 ? (1 << 19) - 1 : x;
+
+ private static int Next(int state, int y) => ZpaqTables.StateTable[state * 4 + y];
+
+ private static int CmInit(int state) {
+ var st = ZpaqTables.StateTable;
+ return ((st[state * 4 + 3] * 2 + 1) << 22) / (st[state * 4 + 2] + st[state * 4 + 3] + 1);
+ }
+
+ private static void Train(Component cr, int y) {
+ ref var pn = ref cr.Cm[cr.Cxt & (uint)(cr.Cm.Length - 1)];
+ var count = pn & 0x3ff;
+ var error = y * 32767 - (int)(pn >> 17);
+ pn += (uint)((error * ZpaqTables.CmDivisor[count] & -1024) + (count < cr.Limit ? 1 : 0));
+ }
+
+ ///
+ /// Finds the 16-byte row of for context , replacing the
+ /// least used of three candidate rows when none matches.
+ ///
+ private static uint Find(byte[] ht, int sizebits, uint cxt) {
+ var chk = (byte)(cxt >> sizebits);
+ var h0 = (cxt * 16) & (uint)(ht.Length - 16);
+ if (ht[h0] == chk)
+ return h0;
+ var h1 = h0 ^ 16;
+ if (ht[h1] == chk)
+ return h1;
+ var h2 = h0 ^ 32;
+ if (ht[h2] == chk)
+ return h2;
+ uint pick;
+ if (ht[h0 + 1] <= ht[h1 + 1] && ht[h0 + 1] <= ht[h2 + 1])
+ pick = h0;
+ else if (ht[h1 + 1] < ht[h2 + 1])
+ pick = h1;
+ else
+ pick = h2;
+ ht.AsSpan((int)pick, 16).Clear();
+ ht[pick] = chk;
+ return pick;
+ }
+
+ private static ushort[] BuildSquash() {
+ var t = new ushort[4096];
+ ZpaqTables.SquashMiddle.CopyTo(t, 1376);
+ for (var i = 2720; i < 4096; ++i)
+ t[i] = 32767;
+ return t;
+ }
+
+ private static short[] BuildStretch() {
+ var t = new short[32768];
+ var k = 16384;
+ var runs = ZpaqTables.StretchRunLengths;
+ for (var i = 0; i < 712; ++i)
+ for (int j = runs[i]; j > 0; --j)
+ t[k++] = (short)i;
+ for (var i = 0; i < 16384; ++i)
+ t[i] = (short)-t[32767 - i];
+ return t;
+ }
+}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqRangeCoder.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqRangeCoder.cs
deleted file mode 100644
index 091f584f3..000000000
--- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqRangeCoder.cs
+++ /dev/null
@@ -1,232 +0,0 @@
-namespace FileFormat.Zpaq;
-
-///
-/// Shared constants and the safety argument of the ZPAQ binary range coder.
-///
-///
-///
-/// The coder is a carry-propagating binary range coder. It carries a 32-bit
-/// range and an arbitrarily long low, of which only the pending
-/// tail is held in memory: one cached byte plus a count of deferred 0xFF
-/// bytes that a later carry may still turn into 0x00. Splitting the range
-/// by bound = (range / 2^16) * p assigns the lower subrange to a 1 bit and
-/// the upper subrange to a 0 bit, so the more confident the model is that a 1
-/// follows, the cheaper coding that 1 becomes.
-///
-/// Invariant. On entry to every bit,
-/// 2^24 <= range <= 2^32 - 1 and 1 <= p <= 65535. Both
-/// subranges are then non-empty, with width at least 256:
-///
-///
-/// -
-/// the 1 subrange has width bound = (range / 2^16) * p >= (2^24 / 2^16) * 1 = 256;
-///
-/// -
-/// the 0 subrange has width
-/// range - bound >= range - (range / 2^16) * 65535
-/// = range - (range / 2^16) * 2^16 + (range / 2^16)
-/// >= (range / 2^16) >= 256,
-/// because range - (range / 2^16) * 2^16 is the non-negative remainder
-/// of the integer division.
-///
-///
-///
-/// Renormalisation multiplies range by 256 while it is below 2^24. Since it
-/// is never zero it reaches 2^24 in at most two steps, and since each step starts
-/// from a value below 2^24 the result stays below 2^32, so the invariant is
-/// restored rather than merely approached. The interval can therefore never
-/// collapse to zero width, and the decoder — which performs the identical integer
-/// arithmetic — always finds its code word strictly inside exactly one subrange.
-///
-///
-/// The bound also keeps low small enough to compute exactly:
-/// low + range < 2^33 holds initially and is preserved by both coding
-/// (which never increases the sum) and renormalisation (where
-/// (low mod 2^24) + range < 2^25 before the common factor of 256), so
-/// low never needs more than 33 bits and the carry out of bit 32 is always
-/// a single bit.
-///
-///
-/// Reference: Matt Mahoney, The ZPAQ Open Standard Format for Highly Compressed
-/// Data (level 1/2 specification), section on arithmetic coding; and
-/// Data Compression Explained, chapter 4, on binary arithmetic coding with
-/// carry counting.
-///
-///
-public static class ZpaqRangeCoder {
-
- /// Lower bound of the normalised range; the coder renormalises below this.
- public const uint RangeMinimum = 0x01000000u;
-
- /// Value at or above which the top byte of low may still be bumped by a carry.
- public const uint CarryEdge = 0xFF000000u;
-
- /// Number of fractional bits in a probability, so probabilities span 0..2^16.
- public const int ProbabilityBits = 16;
-
- /// Smallest probability of a 1 bit the coder accepts.
- public const int MinimumProbability = 1;
-
- /// Largest probability of a 1 bit the coder accepts.
- public const int MaximumProbability = 65535;
-
- /// Number of low bytes the encoder pushes out when the message ends.
- public const int FlushBytes = 5;
-
- ///
- /// Computes the split point between the 1 subrange and the 0 subrange.
- ///
- /// The current range, at least .
- /// Probability of a 1 bit, clamped into 1..65535.
- /// The width of the 1 subrange, at least 256 and at least 256 below .
- public static uint Split(uint range, int probabilityOfOne) {
- var p = (uint)Math.Clamp(probabilityOfOne, MinimumProbability, MaximumProbability);
- return (range >> ProbabilityBits) * p;
- }
-}
-
-///
-/// Carry-propagating binary range encoder. See for the
-/// invariant that keeps both subranges non-empty.
-///
-public sealed class ZpaqRangeEncoder {
-
- private readonly Stream _output;
- private ulong _low;
- private uint _range = uint.MaxValue;
- private byte _cache;
- private int _pending = 1;
-
- ///
- /// Creates an encoder writing its bytes to the given stream.
- ///
- /// Destination for the coded bytes.
- /// Thrown if is null.
- public ZpaqRangeEncoder(Stream output) {
- ArgumentNullException.ThrowIfNull(output);
- this._output = output;
- }
-
- /// Gets the current range, for tests that assert the coder invariant.
- public uint Range => this._range;
-
- ///
- /// Codes one bit against the given probability.
- ///
- /// The bit to code; zero or non-zero.
- /// Probability that the bit is 1, clamped into 1..65535.
- public void EncodeBit(int bit, int probabilityOfOne) {
- var bound = ZpaqRangeCoder.Split(this._range, probabilityOfOne);
- if (bit != 0)
- this._range = bound;
- else {
- this._low += bound;
- this._range -= bound;
- }
-
- while (this._range < ZpaqRangeCoder.RangeMinimum) {
- this.ShiftLow();
- this._range <<= 8;
- }
- }
-
- ///
- /// Pushes out the remaining bytes of low, ending the coded stream.
- ///
- public void Flush() {
- for (var i = 0; i < ZpaqRangeCoder.FlushBytes; ++i)
- this.ShiftLow();
- }
-
- ///
- /// Emits the top byte of low, deferring it while a later carry could still
- /// increment it, and shifts low up by one byte.
- ///
- private void ShiftLow() {
- var carry = (uint)(this._low >> 32);
- var value = (uint)this._low;
-
- // While the top byte is 0xFF a carry would ripple through it, so it is only
- // counted, not emitted. Once a byte arrives that either cannot carry or has
- // just carried, the whole deferred run is resolved at once.
- if (value < ZpaqRangeCoder.CarryEdge || carry != 0) {
- var held = this._cache;
- do {
- this._output.WriteByte((byte)(held + carry));
- held = 0xFF;
- --this._pending;
- } while (this._pending != 0);
- this._cache = (byte)(value >> 24);
- }
-
- ++this._pending;
- this._low = (ulong)(value & 0x00FFFFFFu) << 8;
- }
-}
-
-///
-/// Carry-propagating binary range decoder, the exact mirror of
-/// .
-///
-public sealed class ZpaqRangeDecoder {
-
- private readonly byte[] _input;
- private int _position;
- private uint _range = uint.MaxValue;
- private uint _code;
-
- ///
- /// Creates a decoder reading from the given buffer, priming its code word from
- /// the first bytes.
- ///
- /// Buffer holding the coded bytes.
- /// Index of the first coded byte.
- /// Thrown if is null.
- public ZpaqRangeDecoder(byte[] input, int offset) {
- ArgumentNullException.ThrowIfNull(input);
- this._input = input;
- this._position = offset;
-
- // The encoder's first flushed byte is always the initial empty cache, so the
- // code word is the five leading bytes read big-endian into 32 bits.
- for (var i = 0; i < ZpaqRangeCoder.FlushBytes; ++i)
- this._code = (this._code << 8) | this.NextByte();
- }
-
- /// Gets the current range, for tests that assert the coder invariant.
- public uint Range => this._range;
-
- ///
- /// Decodes one bit against the given probability.
- ///
- /// Probability that the bit is 1, clamped into 1..65535.
- /// The decoded bit, 0 or 1.
- public int DecodeBit(int probabilityOfOne) {
- var bound = ZpaqRangeCoder.Split(this._range, probabilityOfOne);
-
- int bit;
- if (this._code < bound) {
- bit = 1;
- this._range = bound;
- } else {
- bit = 0;
- this._code -= bound;
- this._range -= bound;
- }
-
- while (this._range < ZpaqRangeCoder.RangeMinimum) {
- this._range <<= 8;
- this._code = (this._code << 8) | this.NextByte();
- }
-
- return bit;
- }
-
- ///
- /// Reads the next coded byte, or zero past the end of the buffer. Past the end
- /// the encoder's flush has already pinned every remaining bit, so the padding
- /// value only has to be the same on both sides.
- ///
- private uint NextByte() =>
- this._position < this._input.Length ? this._input[this._position++] : 0u;
-}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqReader.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqReader.cs
index ff3a50a78..bec85b951 100644
--- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqReader.cs
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqReader.cs
@@ -1,576 +1,523 @@
+using System.Buffers.Binary;
using System.Text;
namespace FileFormat.Zpaq;
///
-/// Reads the journaling index of a ZPAQ archive, exposing the files recorded
-/// across all transactions.
+/// Reads ZPAQ archives in both layouts the format defines: the journaling layout of
+/// zpaq 6.x/7.x (versioned transactions of c/d/h/i blocks with
+/// deduplicated, SHA-1 checked fragments) and the plain streaming layout (one file per
+/// block or segment run).
///
///
-/// ZPAQ archives contain a ZPAQL virtual-machine program that is required to
-/// decompress the payload. This reader parses only the journaling structure
-/// (block headers, transaction metadata) to enumerate entries and their
-/// attributes. Decompression is not supported; always
-/// throws .
+/// The archive walk follows Jidac::read_archive of zpaq 7.15 (Matt Mahoney, public
+/// domain); block decoding is the libzpaq conversion in this namespace. Journaling block
+/// names are jDC<YYYYMMDDHHMMSS><type><number:10> and their segment comments
+/// end in "jDC\x01". Encrypted archives (AES-256 with a 32-byte salt) are refused.
///
public sealed class ZpaqReader : IDisposable {
- private readonly Stream _stream;
- private readonly bool _leaveOpen;
+ private static ReadOnlySpan JournalMarker => "jDC\x01"u8;
+
+ private readonly Stream _stream;
+ private readonly bool _leaveOpen;
+ private readonly List _ht = [new([], -2)];
+ private readonly List _blocks = [];
+ private readonly List _versions = [];
+ private readonly List _log = [];
private readonly List _entries = [];
- // Captured uncompressed data-block bytes keyed by filename (last write wins),
- // populated for STORED data blocks so this reader can extract archives that
- // were written without ZPAQL compression (the layout ZpaqWriter emits).
- private readonly Dictionary _dataByName = new(StringComparer.Ordinal);
- private bool _disposed;
-
- ///
- /// Gets the entries discovered by scanning the archive journal.
- /// The list reflects the last recorded state of each file across all
- /// transactions (i.e. later transactions that mention the same filename
- /// supersede earlier ones in the final view).
- ///
- public IReadOnlyList Entries => _entries;
-
- // ── Construction ─────────────────────────────────────────────────────────
-
- ///
- /// Opens a ZPAQ archive stream and scans its journal.
- ///
- ///
- /// A readable stream positioned at the beginning of a ZPAQ archive.
- /// The stream does not need to be seekable; the reader performs a single
- /// forward scan.
- ///
- ///
- /// to leave open when
- /// this reader is disposed; (default) to dispose it.
- ///
- ///
- /// Thrown when is .
- ///
- ///
- /// Thrown when is not readable.
- ///
- ///
- /// Thrown when the stream contains no recognizable ZPAQ blocks.
- ///
+ private int _cachedBlock = -1;
+ private byte[] _cachedData = [];
+ private bool _disposed;
+
+ private sealed record Fragment(byte[] Sha1, int Usize);
+
+ private sealed class Block(uint start, long offset) {
+ public uint Start = start;
+ public long Offset = offset;
+ public long Usize = -1;
+ public long Bsize;
+ public uint Frags;
+ }
+
+ private sealed record VersionInfo(long Date, long CompressedSize);
+
+ private sealed record IndexRecord(int Version, string Name, long Date, long Attributes, List Fragments, List<(long, int)> Segments);
+
+ /// Opens a ZPAQ archive and reads its index.
+ /// A readable stream positioned at the start of the archive.
+ /// Whether to leave open on dispose.
+ /// The data is not a ZPAQ archive.
+ /// The archive is encrypted.
public ZpaqReader(Stream stream, bool leaveOpen = false) {
ArgumentNullException.ThrowIfNull(stream);
if (!stream.CanRead)
throw new ArgumentException("Stream must be readable.", nameof(stream));
-
- _stream = stream;
- _leaveOpen = leaveOpen;
-
- ScanJournal();
- }
-
- // ── Journal scanning ─────────────────────────────────────────────────────
-
- ///
- /// Scans the stream for ZPAQ blocks and builds the entry list from any
- /// header ('c') blocks whose content is accessible (uncompressed or stored).
- ///
- private void ScanJournal() {
- // We maintain a byte-level sliding window to detect the 3-byte "zPQ"
- // prefix even when the stream is not seekable.
- // For each block we find:
- // • record the block's raw byte span (for CompressedSize accounting)
- // • if it is a header ('c') block, attempt to parse file metadata
- // • group blocks by transaction number
-
- var raw = new BufferedStream(_stream, 65536);
- var rawBytes = new RawByteReader(raw);
-
- // Map from filename → latest entry (journaling: last write wins).
- var latestByName = new Dictionary(StringComparer.Ordinal);
- // Cumulative data-block bytes associated with the current transaction.
- long txDataBytes = 0;
- var txVersion = 0;
- // Names collected from the current transaction's header block.
- var txNames = new List<(string Name, long Size, DateTime? Modified)>();
- // Raw STORED bytes from the current transaction's data ('d') block(s).
- var txData = new List();
- // Running block-start position (approximate; only accurate when seekable).
- long blockStart = 0;
- var foundAnyBlock = false;
-
- while (true) {
- // Scan forward until we find "zPQ".
- if (!rawBytes.ScanToPrefix(ZpaqConstants.BlockPrefix))
- break;
-
- foundAnyBlock = true;
- blockStart = rawBytes.Position - 3; // prefix length
-
- // Read level byte and block-type byte.
- var lvl = rawBytes.ReadByte();
- if (lvl < 0) break;
- var type = rawBytes.ReadByte();
- if (type < 0) break;
-
- switch ((byte)type) {
- case ZpaqConstants.BlockTypeHeader:
- // 'c' block — start of a new transaction.
- // Flush the previous transaction into the entry list.
- FlushTransaction(latestByName, txNames, txDataBytes, txVersion, txData, _dataByName);
-
- txVersion++;
- txDataBytes = 0;
- txNames.Clear();
- txData.Clear();
-
- // A header block in a level-1 archive that was created with
- // "zpaq add" uses stored (uncompressed) content so that the
- // journal can be scanned without decompressing. The block
- // payload immediately follows the 5-byte block header and is
- // terminated by a 0xFF byte (end-of-block marker in ZPAQ1).
- ParseHeaderBlockPayload(rawBytes, txNames);
- break;
-
- case ZpaqConstants.BlockTypeData:
- // 'd' block — data payload. Capture the STORED bytes so Extract can
- // return them; MeasureBlockPayload's accounting comes from the count.
- var captured = CaptureBlockPayload(rawBytes);
- txData.AddRange(captured);
- txDataBytes += captured.Length;
- break;
-
- case ZpaqConstants.BlockTypeIndex:
- // 'h' block — transaction close / hash block. Flush.
- FlushTransaction(latestByName, txNames, txDataBytes, txVersion, txData, _dataByName);
- txVersion++;
- txDataBytes = 0;
- txNames.Clear();
- txData.Clear();
- SkipBlockPayload(rawBytes);
- break;
-
- case ZpaqConstants.BlockTypeCompressed:
- default:
- // Unknown or compressed data block — skip.
- SkipBlockPayload(rawBytes);
- break;
- }
+ this._leaveOpen = leaveOpen;
+ if (stream.CanSeek)
+ this._stream = stream;
+ else {
+ var copy = new MemoryStream();
+ stream.CopyTo(copy);
+ copy.Position = 0;
+ this._stream = copy;
+ if (!leaveOpen)
+ stream.Dispose();
+ this._leaveOpen = false;
}
-
- // Flush any trailing transaction that had no 'h' block.
- FlushTransaction(latestByName, txNames, txDataBytes, txVersion, txData, _dataByName);
-
- if (!foundAnyBlock)
- ThrowInvalidData("No ZPAQ blocks found in stream.");
-
- // Build the final entry list: latest state of each file.
- _entries.AddRange(latestByName.Values
- .OrderBy(e => e.FileName, StringComparer.Ordinal));
+ this.ReadArchive();
+ this._entries.AddRange(this.Snapshot(int.MaxValue));
}
- // ── Header block parsing ─────────────────────────────────────────────────
-
- ///
- /// Attempts to parse the content of a ZPAQ level-1 journaling 'c' block.
- ///
- /// A stored (not-ZPAQL-compressed) 'c' block has the layout:
- /// 8 bytes : Windows FILETIME of the transaction date/time
- /// For each file in the transaction:
- /// 1 byte : attribute flags (0x20 = directory on Windows,
- /// 0x10 = directory on Unix)
- /// variable : null-terminated UTF-8 filename
- /// 8 bytes : uncompressed file size (little-endian int64)
- /// 0xFF : end-of-block marker
- ///
- /// If the first byte of the payload is NOT a valid attribute byte
- /// (i.e. it is 0xFF, meaning an immediately empty block, or it
- /// indicates a ZPAQL program header) we fall through gracefully.
- ///
- private static void ParseHeaderBlockPayload(
- RawByteReader reader,
- List<(string Name, long Size, DateTime? Modified)> names) {
-
- // The block payload may be preceded by a 2-byte ZPAQL program length
- // (big-endian) in ZPAQ level-1 compressed blocks. In journaling archives
- // created by the reference `zpaq` tool the 'c' block content is stored
- // verbatim (not ZPAQL-compressed) and starts immediately with the
- // 8-byte Windows FILETIME.
-
- // Read the 8-byte transaction timestamp (Windows FILETIME, little-endian).
- Span tsBytes = stackalloc byte[8];
- if (!reader.TryReadBytes(tsBytes))
- return; // truncated — give up
-
- var ft = BitConverter.ToInt64(tsBytes);
- var txDate = DecodeWindowsFileTime(ft);
-
- // Read file entries until we hit 0xFF (end-of-block).
- Span szBytes = stackalloc byte[8]; // moved outside loop to satisfy CA2014
- while (true) {
- var attr = reader.ReadByte();
- if (attr < 0 || attr == 0xFF)
- break; // EOF or end-of-block marker
-
- // Read null-terminated UTF-8 filename.
- var name = reader.ReadNullTerminatedString();
- if (name.Length == 0) {
- // Skip the 8-byte size field and continue.
- reader.Skip(8);
- continue;
- }
+ /// Files and directories present in the latest version, ordered by name.
+ public IReadOnlyList Entries => this._entries;
- // Read 8-byte uncompressed size.
- if (!reader.TryReadBytes(szBytes))
- break;
- var size = BitConverter.ToInt64(szBytes);
+ /// Number of versions (transactions) in the archive.
+ public int VersionCount => this._versions.Count;
- names.Add((name, size, txDate));
- }
- }
+ /// Number of blocks that failed to decode and were skipped while reading the index.
+ public int Errors { get; private set; }
- // ── Block payload helpers ─────────────────────────────────────────────────
-
- ///
- /// Measures the byte length of the current block payload by scanning
- /// forward to the next "zPQ" prefix (or end of stream) without consuming
- /// the prefix itself. Returns the number of payload bytes consumed.
- ///
- private static long MeasureBlockPayload(RawByteReader reader) {
- var start = reader.Position;
- reader.SkipToNextBlock();
- return reader.Position - start;
+ /// UTC date of version (1-based), or for a streaming archive.
+ public DateTime? GetVersionDate(int version) {
+ ArgumentOutOfRangeException.ThrowIfLessThan(version, 1);
+ ArgumentOutOfRangeException.ThrowIfGreaterThan(version, this._versions.Count);
+ return ZpaqDate.ToDateTime(this._versions[version - 1].Date);
}
- ///
- /// Captures the current block's payload bytes (everything up to, but not
- /// including, the next "zPQ" prefix or end of stream). Used for STORED data
- /// blocks so their raw bytes can be returned by .
- ///
- private static byte[] CaptureBlockPayload(RawByteReader reader) =>
- reader.CaptureToNextBlock();
-
- ///
- /// Skips the current block payload, positioning the reader just before
- /// the next "zPQ" prefix (or at end of stream).
- ///
- private static void SkipBlockPayload(RawByteReader reader) =>
- reader.SkipToNextBlock();
-
- // ── Transaction flushing ─────────────────────────────────────────────────
-
- private static void FlushTransaction(
- Dictionary dest,
- List<(string Name, long Size, DateTime? Modified)> names,
- long dataBytes,
- int version,
- List data,
- Dictionary dataByName) {
- if (names.Count == 0)
- return;
-
- // Distribute the data-block bytes equally across files in the transaction
- // as a rough approximation (ZPAQ doesn't store per-file compressed sizes).
- var perFile = names.Count > 0 ? dataBytes / names.Count : 0;
-
- // Slice the captured STORED transaction bytes per file in header order using
- // each file's recorded size. Only valid for the uncompressed layout this
- // toolkit's writer emits; ZPAQL-compressed blocks leave dataByName unset and
- // Extract surfaces them as empty.
- var dataOffset = 0;
- foreach (var (name, size, modified) in names) {
- var isDir = name.EndsWith('/') || name.EndsWith('\\');
- var entry = new ZpaqEntry(
- fileName: NormalizeName(name),
- size: size,
- compressedSize: isDir ? 0 : perFile,
- lastModified: modified,
- isDirectory: isDir,
- version: version);
-
- dest[entry.FileName] = entry; // later transaction supersedes earlier
-
- if (!isDir && size >= 0 && dataOffset + size <= data.Count) {
- var slice = new byte[size];
- data.CopyTo(dataOffset, slice, 0, (int)size);
- dataByName[entry.FileName] = slice; // later transaction supersedes earlier
- dataOffset += (int)size;
- }
- }
+ /// The archive contents as of version (1-based).
+ public IReadOnlyList GetEntries(int version) {
+ ArgumentOutOfRangeException.ThrowIfLessThan(version, 1);
+ ArgumentOutOfRangeException.ThrowIfGreaterThan(version, this._versions.Count);
+ return this.Snapshot(version);
}
- // ── Utilities ─────────────────────────────────────────────────────────────
-
- private static string NormalizeName(string name) =>
- name.Replace('\\', '/');
+ /// Returns the entry's content as a read-only stream.
+ public Stream Extract(ZpaqEntry entry) => new MemoryStream(this.ExtractBytes(entry), writable: false);
- private static DateTime? DecodeWindowsFileTime(long ft) {
- if (ft <= 0)
- return null;
-
- try {
- var ticks = ft - ZpaqConstants.WindowsToUnixEpochTicks;
- if (ticks < 0)
- return null;
- return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)
- .AddTicks(ticks);
- } catch {
- return null;
- }
- }
-
- private static void ThrowInvalidData(string message) =>
- throw new InvalidDataException(message);
-
- // ── Extract (not supported) ───────────────────────────────────────────────
-
- ///
- /// Returns the entry's uncompressed bytes as a stream when the data block was
- /// STORED (the layout this toolkit's emits).
- ///
- ///
- /// General ZPAQL-compressed archives require executing a ZPAQL virtual machine
- /// which is out of scope; for those the captured-data map is empty and this
- /// method throws .
- ///
- public Stream Extract(ZpaqEntry entry) {
+ /// Decodes the entry's content, verifying every fragment's (or segment's) SHA-1.
+ /// The data is corrupt or a checksum does not match.
+ public byte[] ExtractBytes(ZpaqEntry entry) {
ArgumentNullException.ThrowIfNull(entry);
+ ObjectDisposedException.ThrowIf(this._disposed, this);
if (entry.IsDirectory)
- return new MemoryStream([], writable: false);
- if (_dataByName.TryGetValue(entry.FileName, out var bytes))
- return new MemoryStream(bytes, writable: false);
- throw new NotSupportedException(
- "ZPAQ decompression of ZPAQL-compressed blocks requires a ZPAQL virtual machine which is not implemented.");
+ return [];
+ return entry.Segments.Count > 0 ? this.ExtractStreaming(entry) : this.ExtractJournaling(entry);
}
- // ── IDisposable ───────────────────────────────────────────────────────────
-
- ///
+ ///
public void Dispose() {
- if (!_disposed) {
- _disposed = true;
- if (!_leaveOpen)
- _stream.Dispose();
- }
+ if (this._disposed)
+ return;
+ this._disposed = true;
+ if (!this._leaveOpen)
+ this._stream.Dispose();
}
-}
-
-// ── Internal streaming helper ─────────────────────────────────────────────────
-///
-/// A forward-only, low-allocation byte reader that supports the scanning
-/// operations needed by .
-///
-internal sealed class RawByteReader {
- private readonly Stream _stream;
- private readonly byte[] _buf;
- private int _pos; // read position within _buf
- private int _len; // valid bytes in _buf
- private long _streamOffset; // bytes consumed from stream so far
-
- // The "zPQ" prefix as a cached value.
- private static ReadOnlySpan Prefix => ZpaqConstants.BlockPrefix;
- private const int PrefixLen = 3;
-
- // Lookahead window — we keep the last (PrefixLen - 1) bytes of each chunk
- // in the buffer overlap so we never miss a prefix that straddles a chunk
- // boundary.
- private const int BufSize = 65536;
- private const int Overlap = PrefixLen - 1; // = 2
-
- internal RawByteReader(Stream stream) {
- _stream = stream;
- _buf = new byte[BufSize + Overlap];
- _pos = Overlap; // first Overlap bytes reserved for tail-of-previous-chunk
- _len = Overlap; // nothing real yet
+ // ── index ──────────────────────────────────────────────────────────────────
+
+ private void ReadArchive() {
+ var s = this._stream;
+ s.Position = 0;
+ Span head = stackalloc byte[4];
+ var nr = s.ReadAtLeast(head, 4, throwOnEndOfStream: false);
+ if (nr > 0 && !head[..nr].SequenceEqual("7kSt"u8[..nr]) && (nr < 4 || !head[..3].SequenceEqual("zPQ"u8) || head[3] < 1)) {
+ if (nr == 4 && head[..3].SequenceEqual("zPQ"u8))
+ throw new InvalidDataException("ZPAQ: unsupported block level.");
+ throw new InvalidDataException("Not a ZPAQ archive (or an encrypted one, which needs the password-derived key).");
+ }
+ s.Position = 0;
+
+ var lastFile = "";
+ long blockOffset = 0;
+ long dataOffset = 0;
+ var foundData = false;
+ var first = true;
+ var done = false;
+ var dt = new Dictionary(StringComparer.Ordinal);
+
+ while (!done) {
+ var d = new ZpaqDecompresser(new ZpaqStreamReader(s));
+ var jumped = false;
+ try {
+ while (!jumped && d.FindBlock()) {
+ foundData = true;
+ var segs = 0;
+ var skip = false;
+ string? filename;
+ while ((filename = d.FindFilename()) != null) {
+ if (filename.Length > 0)
+ lastFile = filename.Replace('\\', '/');
+ var comment = d.ReadComment();
+
+ if (comment.AsSpan().EndsWith(JournalMarker)) {
+ if (Encoding.UTF8.GetByteCount(filename) != 28 || !filename.StartsWith("jDC", StringComparison.Ordinal))
+ ZpaqError.Throw("bad journaling block name");
+ if (skip)
+ ZpaqError.Throw("mixed journaling and streaming block");
+
+ long usize = 0;
+ for (var i = 0; i < comment.Length && char.IsAsciiDigit((char)comment[i]); ++i) {
+ usize = usize * 10 + comment[i] - '0';
+ if (usize > 0xffffffff)
+ ZpaqError.Throw("journaling block too big");
+ }
+ long fdate = 0, num = 0;
+ int k;
+ for (k = 3; k < 17 && char.IsAsciiDigit(filename[k]); ++k)
+ fdate = fdate * 10 + filename[k] - '0';
+ if (k != 17 || fdate < 19000000000000L || fdate >= 30000000000000L)
+ ZpaqError.Throw("bad date");
+ for (k = 18; k < 28 && char.IsAsciiDigit(filename[k]); ++k)
+ num = num * 10 + filename[k] - '0';
+ if (k != 28 || num > 0xffffffff)
+ ZpaqError.Throw("bad fragment");
+
+ var type = filename[17];
+ var os = new ZpaqBuffer((int)Math.Min(usize, 1 << 20)) { Limit = usize };
+ if (type is 'c' or 'h' or 'i') {
+ var sha1 = new ZpaqSha1();
+ d.SetOutput(os);
+ d.SetSha1(sha1);
+ d.Decompress();
+ var stored = d.ReadSegmentEnd();
+ d.SetOutput(null);
+ d.SetSha1(null);
+ if (os.Size != usize)
+ ZpaqError.Throw("bad block size");
+ if (usize != sha1.Size)
+ ZpaqError.Throw("bad checksum size");
+ if (stored != null && !stored.AsSpan().SequenceEqual(sha1.Result()))
+ ZpaqError.Throw("bad checksum");
+ } else
+ d.ReadSegmentEnd();
+
+ switch (type) {
+ case 'c': {
+ if (os.Size < 8)
+ ZpaqError.Throw("c block too small");
+ dataOffset = s.Position + 1 - d.Buffered;
+ var jmp = BinaryPrimitives.ReadInt64LittleEndian(os.Span);
+ if (jmp < 0) {
+ done = true; // incomplete transaction: roll back to here
+ jumped = true;
+ break;
+ }
+ this._versions.Add(new(fdate, jmp));
+ if (jmp > 0) {
+ s.Position = dataOffset + jmp;
+ jumped = true;
+ }
+ break;
+ }
+ case 'h': {
+ if (this._versions.Count == 0)
+ ZpaqError.Throw("fragment table before transaction header");
+ if (os.Size % 24 != 4)
+ ZpaqError.Throw("bad h block size");
+ var n = (uint)((os.Size - 4) / 24);
+ if (num < 1 || num + n > 0xffffffff)
+ ZpaqError.Throw("bad h fragment");
+ var span = os.Span;
+ var bsize = BinaryPrimitives.ReadUInt32LittleEndian(span);
+ var p = 4;
+ for (var i = 0u; i < n; ++i) {
+ if (i == 0)
+ this._blocks.Add(new((uint)num, dataOffset) { Usize = 8, Bsize = bsize, Frags = n });
+ while (this._ht.Count <= num + i)
+ this._ht.Add(new([], -2));
+ var hash = span.Slice(p, 20).ToArray();
+ p += 20;
+ var f = BinaryPrimitives.ReadUInt32LittleEndian(span[p..]);
+ p += 4;
+ if (f > 0x7fffffff)
+ ZpaqError.Throw("fragment too big");
+ this._ht[(int)(num + i)] = new(hash, (int)f);
+ this._blocks[^1].Usize += f + 4u;
+ }
+ dataOffset += bsize;
+ break;
+ }
+ case 'i': {
+ if (this._versions.Count == 0)
+ ZpaqError.Throw("index before transaction header");
+ this.ParseIndex(os.Span, dt);
+ break;
+ }
+ case 'd':
+ break;
+ default:
+ ZpaqError.Throw("unexpected journaling block");
+ break;
+ }
+ if (jumped)
+ break;
+ } else {
+ // Streaming format: each segment is (a part of) a file.
+ if (this._versions.Count == 0)
+ this._versions.Add(new(0, -1));
+ d.ReadSegmentEnd();
+ skip = true;
+ var fn = lastFile;
+ if (filename.Length > 0 || first || !dt.TryGetValue(fn, out var rec)) {
+ rec = new(this._versions.Count, fn, ParseStreamingDate(comment), ParseStreamingAttributes(comment), [], []);
+ dt[fn] = rec;
+ this._log.Add(rec);
+ }
+ rec.Segments.Add((blockOffset, segs));
+ if (segs == 0 || this._blocks.Count == 0)
+ this._blocks.Add(new((uint)this._ht.Count, blockOffset));
+ this._ht.Add(new([], -1));
+ }
+ ++segs;
+ first = false;
+ }
+ if (!jumped)
+ blockOffset = s.Position - d.Buffered;
+ }
+ if (!jumped)
+ done = true;
+ } catch (InvalidDataException) when (foundData) {
+ // Like zpaq: report, then resume scanning for the next block after the bad one.
+ ++this.Errors;
+ s.Position -= d.Buffered;
+ }
+ }
+ if (s.Length > 0 && !foundData)
+ throw new InvalidDataException("ZPAQ: archive contains no data.");
}
- /// Gets the number of bytes consumed from the stream so far.
- internal long Position => _streamOffset - (_len - _pos);
-
- // ── Primitive I/O ────────────────────────────────────────────────────────
-
- /// Returns the next byte or -1 on end-of-stream.
- internal int ReadByte() {
- if (!EnsureAvailable(1))
- return -1;
- return _buf[_pos++];
+ private void ParseIndex(ReadOnlySpan s, Dictionary dt) {
+ var p = 0;
+ var version = this._versions.Count;
+ while (p + 9 <= s.Length) {
+ var date = BinaryPrimitives.ReadInt64LittleEndian(s[p..]);
+ p += 8;
+ var len = s[p..].IndexOf((byte)0);
+ if (len < 0)
+ ZpaqError.Throw("filename too long");
+ if (len > 65535)
+ ZpaqError.Throw("filename too long");
+ var name = Encoding.UTF8.GetString(s.Slice(p, len));
+ p += len + 1;
+ long attr = 0;
+ var ptr = new List();
+ if (date != 0) {
+ if (p + 4 > s.Length)
+ ZpaqError.Throw("missing attr");
+ var na = BinaryPrimitives.ReadUInt32LittleEndian(s[p..]);
+ p += 4;
+ if (p + na > s.Length || na > 65535)
+ ZpaqError.Throw("attr too long");
+ for (var i = 0; i < na; ++i, ++p)
+ if (i < 8)
+ attr += (long)s[p] << (i * 8);
+ if (p + 4 > s.Length)
+ ZpaqError.Throw("missing ptr");
+ var ni = BinaryPrimitives.ReadUInt32LittleEndian(s[p..]);
+ p += 4;
+ if (ni > (s.Length - p) / 4u)
+ ZpaqError.Throw("ptr list too long");
+ for (var i = 0u; i < ni; ++i, p += 4)
+ ptr.Add(BinaryPrimitives.ReadUInt32LittleEndian(s[p..]));
+ }
+ var rec = new IndexRecord(version, name, date, attr, ptr, []);
+ dt[name] = rec;
+ this._log.Add(rec);
+ }
}
- ///
- /// Reads exactly .Length bytes into .
- /// Returns if fewer bytes are available.
- ///
- internal bool TryReadBytes(Span dest) {
- var need = dest.Length;
- if (!EnsureAvailable(need))
- return false;
-
- _buf.AsSpan(_pos, need).CopyTo(dest);
- _pos += need;
- return true;
+ private static long ParseStreamingDate(byte[] comment) {
+ // Streaming comments are " " with a 14-digit decimal date.
+ long date = 0;
+ var i = 0;
+ for (; i < comment.Length && char.IsAsciiDigit((char)comment[i]); ++i)
+ date = date * 10 + comment[i] - '0';
+ return i == 14 ? date : 0;
}
- /// Skips exactly bytes.
- internal void Skip(int count) {
- while (count > 0) {
- if (!EnsureAvailable(1))
- return;
- var can = Math.Min(count, _len - _pos);
- _pos += can;
- count -= can;
- }
+ private static long ParseStreamingAttributes(byte[] comment) {
+ var sp = Array.IndexOf(comment, (byte)' ');
+ if (sp < 0 || sp + 1 >= comment.Length)
+ return 0;
+ long attr = comment[sp + 1];
+ long value = 0;
+ for (var i = sp + 2; i < comment.Length && char.IsAsciiDigit((char)comment[i]); ++i)
+ value = value * 10 + comment[i] - '0';
+ return attr | value << 8;
}
- // ── Null-terminated string ────────────────────────────────────────────────
-
- internal string ReadNullTerminatedString() {
- var sb = new StringBuilder(32);
- while (true) {
- var b = ReadByte();
- if (b <= 0) // EOF or null terminator
+ private List Snapshot(int version) {
+ var state = new SortedDictionary(StringComparer.Ordinal);
+ foreach (var rec in this._log) {
+ if (rec.Version > version)
break;
- sb.Append((char)b);
+ state[rec.Name] = rec;
}
- return sb.ToString();
+ var result = new List();
+ foreach (var rec in state.Values) {
+ var streaming = rec.Segments.Count > 0;
+ if (!streaming && rec.Date == 0)
+ continue; // deleted in this version
+ long size = 0;
+ long csize = 0;
+ if (streaming)
+ size = -1;
+ else
+ foreach (var j in rec.Fragments) {
+ if (j == 0 || j >= this._ht.Count)
+ continue;
+ var frag = this._ht[(int)j];
+ if (frag.Usize < 0) {
+ size = -1;
+ break;
+ }
+ size += frag.Usize;
+ var b = this.FindBlock(j);
+ if (b >= 0 && this._blocks[b].Usize > 0)
+ csize += this._blocks[b].Bsize * frag.Usize / this._blocks[b].Usize;
+ }
+ result.Add(new(rec.Name, size, csize, ZpaqDate.ToDateTime(rec.Date), rec.Version, rec.Attributes, rec.Fragments, rec.Segments));
+ }
+ return result;
}
- // ── Block-boundary scanning ───────────────────────────────────────────────
-
- ///
- /// Scans forward from the current position until the 3-byte "zPQ" prefix
- /// is found. On return the stream position is just after the prefix.
- /// Returns if the prefix was found;
- /// if end-of-stream was reached first.
- ///
- internal bool ScanToPrefix(ReadOnlySpan prefix) {
- // Sliding 3-byte window (w[0]..w[2]). We advance one byte at a time until
- // the window matches the 3-byte prefix. Using explicit state avoids the
- // edge case where 0x00 is valid data and cannot be used as a sentinel.
- //
- // State: how many leading bytes of the prefix we have matched so far.
- var matched = 0;
-
- while (true) {
- var b = ReadByte();
- if (b < 0)
- return false; // end-of-stream
-
- if (b == prefix[matched]) {
- matched++;
- if (matched == prefix.Length)
- return true; // full prefix found; stream positioned after it
- } else {
- // Mismatch — restart, but check whether this byte starts the prefix.
- matched = (b == prefix[0]) ? 1 : 0;
- }
+ // ── extraction ─────────────────────────────────────────────────────────────
+
+ private int FindBlock(uint fragment) {
+ int lo = 0, hi = this._blocks.Count - 1;
+ while (lo <= hi) {
+ var mid = (lo + hi) >>> 1;
+ var b = this._blocks[mid];
+ if (fragment < b.Start)
+ hi = mid - 1;
+ else if (fragment >= b.Start + b.Frags)
+ lo = mid + 1;
+ else
+ return mid;
}
+ return -1;
}
- ///
- /// Reads the current block's payload bytes, stopping just before the next
- /// "zPQ" block prefix (or at end-of-stream). The prefix is NOT consumed.
- /// Equivalent to but returns the bytes consumed.
- ///
- internal byte[] CaptureToNextBlock() {
- using var ms = new MemoryStream();
- var prefix = Prefix;
-
- while (true) {
- if (!EnsureAvailable(PrefixLen)) {
- // Fewer than PrefixLen bytes remain — they cannot start a new block, so
- // they belong to this payload. Drain them and stop.
- while (_pos < _len) ms.WriteByte(_buf[_pos++]);
- return ms.ToArray();
- }
-
- var available = _len - _pos;
- var window = _buf.AsSpan(_pos, available);
-
- for (var i = 0; i < window.Length; i++) {
- if (window[i] != prefix[0])
- continue;
-
- if (i + PrefixLen - 1 >= window.Length) {
- // Not enough in buffer to confirm — flush up to i, then refill.
- for (var j = 0; j < i; j++) ms.WriteByte(window[j]);
- _pos += i;
- goto refill;
- }
-
- if (window[i + 1] == prefix[1] && window[i + 2] == prefix[2]) {
- for (var j = 0; j < i; j++) ms.WriteByte(window[j]);
- _pos += i; // stop before the next prefix
- return ms.ToArray();
- }
- }
-
- for (var j = 0; j < window.Length; j++) ms.WriteByte(window[j]);
- _pos = _len;
- refill:
- Refill();
- if (_len == _pos)
- return ms.ToArray();
+ private byte[] ExtractJournaling(ZpaqEntry entry) {
+ if (entry.Size < 0)
+ ZpaqError.Throw("file references fragments of unknown size");
+ var output = new byte[entry.Size];
+ var pos = 0;
+ foreach (var j in entry.Fragments) {
+ if (j == 0 || j >= this._ht.Count)
+ ZpaqError.Throw("bad fragment pointer");
+ var frag = this._ht[(int)j];
+ if (frag.Usize == 0)
+ continue;
+ var bi = this.FindBlock(j);
+ if (bi < 0)
+ ZpaqError.Throw($"fragment {j} is in no data block");
+ var data = this.DecodeBlock(bi);
+ var b = this._blocks[bi];
+ long offset = 0;
+ for (var f = b.Start; f < j; ++f)
+ offset += this._ht[(int)f].Usize;
+ data.AsSpan((int)offset, frag.Usize).CopyTo(output.AsSpan(pos));
+ pos += frag.Usize;
}
+ return output;
}
- ///
- /// Advances past the current block's payload, stopping just before the
- /// next "zPQ" block prefix, or at end-of-stream.
- /// The prefix itself is NOT consumed.
- ///
- internal void SkipToNextBlock() {
- var prefix = Prefix;
- // We need to stop before consuming the prefix, so we look-ahead:
- // scan for 'z' then check the following two bytes.
-
- while (true) {
- if (!EnsureAvailable(PrefixLen))
- return; // EOF — nothing more to skip
-
- var available = _len - _pos;
- var window = _buf.AsSpan(_pos, available);
-
- for (var i = 0; i < window.Length; i++) {
- if (window[i] != prefix[0])
- continue;
-
- // Potential match — need two more bytes.
- if (i + PrefixLen - 1 >= window.Length) {
- // Not enough in buffer — advance to i and refill.
- _pos += i;
- goto refill;
- }
+ private byte[] DecodeBlock(int bi) {
+ if (bi == this._cachedBlock)
+ return this._cachedData;
+ var b = this._blocks[bi];
+ long outputSize = 0;
+ for (var f = b.Start; f < b.Start + b.Frags; ++f)
+ outputSize += this._ht[(int)f].Usize;
+
+ this._stream.Position = b.Offset;
+ var d = new ZpaqDecompresser(new ZpaqStreamReader(this._stream));
+ var output = new ZpaqBuffer((int)Math.Min(b.Usize, 1 << 24)) { Limit = b.Usize };
+ d.SetOutput(output);
+ if (!d.FindBlock())
+ ZpaqError.Throw("archive block not found");
+ while (d.FindFilename() != null) {
+ d.ReadComment();
+ d.Decompress();
+ if (output.Size >= outputSize)
+ break;
+ d.ReadSegmentEnd();
+ }
+ if (output.Size < outputSize)
+ ZpaqError.Throw("unexpected end of compressed data");
+
+ var data = output.Data;
+ long q = 0;
+ for (var f = b.Start; f < b.Start + b.Frags; ++f) {
+ var frag = this._ht[(int)f];
+ var hash = Compression.Core.Checksums.Sha1.Compute(data.AsSpan((int)q, frag.Usize));
+ if (!hash.AsSpan().SequenceEqual(frag.Sha1))
+ ZpaqError.Throw($"fragment {f} checksum failed");
+ q += frag.Usize;
+ }
+ this._cachedBlock = bi;
+ this._cachedData = data;
+ return data;
+ }
- if (window[i + 1] == prefix[1] && window[i + 2] == prefix[2]) {
- // Found next prefix — stop here without consuming it.
- _pos += i;
- return;
- }
+ private byte[] ExtractStreaming(ZpaqEntry entry) {
+ var output = new ZpaqBuffer();
+ ZpaqDecompresser? d = null;
+ long openBlock = -1;
+ var nextSegment = 0;
+ foreach (var (blockOffset, segment) in entry.Segments) {
+ if (d == null || openBlock != blockOffset || segment < nextSegment) {
+ this._stream.Position = blockOffset;
+ d = new(new ZpaqStreamReader(this._stream));
+ if (!d.FindBlock())
+ ZpaqError.Throw("archive block not found");
+ openBlock = blockOffset;
+ nextSegment = 0;
+ }
+ while (nextSegment <= segment) {
+ if (d.FindFilename() == null)
+ ZpaqError.Throw("missing segment");
+ d.ReadComment();
+ var target = nextSegment == segment;
+ var sha1 = target ? new ZpaqSha1() : null;
+ d.SetOutput(target ? output : null);
+ d.SetSha1(sha1);
+ d.Decompress();
+ var stored = d.ReadSegmentEnd();
+ d.SetOutput(null);
+ d.SetSha1(null);
+ if (sha1 != null && stored != null && !stored.AsSpan().SequenceEqual(sha1.Result()))
+ ZpaqError.Throw("segment checksum failed");
+ ++nextSegment;
}
-
- _pos = _len; // consume the whole window
- refill:
- Refill();
- if (_len == _pos)
- return;
}
+ return output.ToArray();
}
+}
- // ── Buffer management ─────────────────────────────────────────────────────
-
- private bool EnsureAvailable(int need) {
- if (_len - _pos >= need)
- return true;
- Refill();
- return _len - _pos >= need;
+/// Conversion between ZPAQ's decimal dates (YYYYMMDDHHMMSS, UTC) and .
+internal static class ZpaqDate {
+ /// Encodes a UTC time as a decimal date.
+ public static long FromDateTime(DateTime time) {
+ var t = time.Kind == DateTimeKind.Local ? time.ToUniversalTime() : time;
+ return t.Year * 10000000000L + t.Month * 100000000L + t.Day * 1000000L + t.Hour * 10000L + t.Minute * 100L + t.Second;
}
- private void Refill() {
- // Copy the tail of the current buffer into the overlap area.
- var tail = Math.Min(_len - _pos, Overlap);
- if (tail > 0)
- _buf.AsSpan(_len - tail, tail).CopyTo(_buf.AsSpan(Overlap - tail, tail));
-
- _pos = Overlap - tail; // new read position in the buffer
- var read = _stream.Read(_buf, Overlap, BufSize);
- _len = Overlap + read;
- _streamOffset += read;
+ /// Decodes a decimal date, or when it is not a valid date.
+ public static DateTime? ToDateTime(long date) {
+ if (date <= 0)
+ return null;
+ var year = (int)(date / 10000000000L % 10000);
+ var month = (int)(date / 100000000 % 100);
+ var day = (int)(date / 1000000 % 100);
+ var hour = (int)(date / 10000 % 100);
+ var minute = (int)(date / 100 % 100);
+ var second = (int)(date % 100);
+ try {
+ return new DateTime(year, month, day, hour, minute, second, DateTimeKind.Utc);
+ } catch (ArgumentOutOfRangeException) {
+ return null;
+ }
}
}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqSuffixArray.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqSuffixArray.cs
new file mode 100644
index 000000000..2a260ec58
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqSuffixArray.cs
@@ -0,0 +1,146 @@
+namespace FileFormat.Zpaq;
+
+///
+/// Linear-time suffix array construction by induced sorting (SA-IS).
+///
+///
+/// Implemented from G. Nong, S. Zhang and W. H. Chan, "Two Efficient Algorithms for Linear
+/// Time Suffix Array Construction", IEEE Transactions on Computers 60(10), 2011. Suffixes are
+/// ordered as plain byte strings (a proper prefix sorts first), which is the order a virtual
+/// sentinel smaller than every byte produces and the order libzpaq's BWT and suffix-array
+/// match finder expect.
+///
+internal static class ZpaqSuffixArray {
+ /// Returns the suffix array of .
+ public static int[] Build(ReadOnlySpan text) {
+ var n = text.Length;
+ var sa = new int[n];
+ if (n == 0)
+ return sa;
+ if (n == 1)
+ return sa;
+ var s = new int[n];
+ for (var i = 0; i < n; ++i)
+ s[i] = text[i];
+ Sais(s, sa, n, 256);
+ return sa;
+ }
+
+ /// Sorts the suffixes of [0..n) over alphabet [0..k) into .
+ private static void Sais(int[] s, int[] sa, int n, int k) {
+ if (n == 1) {
+ sa[0] = 0;
+ return;
+ }
+
+ // Suffix types: true = S-type, false = L-type. The virtual sentinel at n is S-type.
+ var t = new bool[n];
+ t[n - 1] = false;
+ for (var i = n - 2; i >= 0; --i)
+ t[i] = s[i] < s[i + 1] || (s[i] == s[i + 1] && t[i + 1]);
+
+ bool IsLms(int i) => i > 0 && t[i] && !t[i - 1];
+
+ var bucket = new int[k];
+ void Buckets(bool end) {
+ Array.Clear(bucket);
+ for (var i = 0; i < n; ++i)
+ ++bucket[s[i]];
+ var sum = 0;
+ for (var c = 0; c < k; ++c) {
+ sum += bucket[c];
+ bucket[c] = end ? sum : sum - bucket[c];
+ }
+ }
+
+ void Induce() {
+ Buckets(false);
+ // The sentinel's predecessor n-1 is L-type and comes first in its bucket.
+ sa[bucket[s[n - 1]]++] = n - 1;
+ for (var i = 0; i < n; ++i) {
+ var j = sa[i] - 1;
+ if (sa[i] > 0 && !t[j])
+ sa[bucket[s[j]]++] = j;
+ }
+ Buckets(true);
+ for (var i = n - 1; i >= 0; --i) {
+ var j = sa[i] - 1;
+ if (sa[i] > 0 && t[j])
+ sa[--bucket[s[j]]] = j;
+ }
+ }
+
+ // Stage 1: place LMS suffixes at bucket ends and induce to sort the LMS substrings.
+ Array.Fill(sa, -1);
+ Buckets(true);
+ for (var i = n - 1; i >= 1; --i)
+ if (IsLms(i))
+ sa[--bucket[s[i]]] = i;
+ Induce();
+
+ // Compact the sorted LMS positions to the front of sa.
+ var m = 0;
+ for (var i = 0; i < n; ++i)
+ if (IsLms(sa[i]))
+ sa[m++] = sa[i];
+ for (var i = m; i < n; ++i)
+ sa[i] = -1;
+
+ // Name the LMS substrings; equal substrings share a name.
+ var name = 0;
+ var prev = -1;
+ for (var i = 0; i < m; ++i) {
+ var pos = sa[i];
+ var diff = prev < 0;
+ if (!diff) {
+ for (var d = 0; ; ++d) {
+ var a = pos + d;
+ var b = prev + d;
+ if (a == n || b == n || s[a] != s[b] || t[a] != t[b]) {
+ diff = true;
+ break;
+ }
+ if (d > 0 && (IsLms(a) || IsLms(b))) {
+ diff = !(IsLms(a) && IsLms(b));
+ break;
+ }
+ }
+ }
+ if (diff) {
+ ++name;
+ prev = pos;
+ }
+ sa[m + (pos >> 1)] = name - 1;
+ }
+ var j2 = n - 1;
+ for (var i = n - 1; i >= m; --i)
+ if (sa[i] >= 0)
+ sa[j2--] = sa[i];
+
+ // Stage 2: sort the reduced problem, recursively when names repeat.
+ var s1 = new int[m];
+ Array.Copy(sa, n - m, s1, 0, m);
+ var sa1 = new int[m];
+ if (name < m)
+ Sais(s1, sa1, m, name);
+ else
+ for (var i = 0; i < m; ++i)
+ sa1[s1[i]] = i;
+
+ // Stage 3: map back to LMS positions in sorted order and induce the full array.
+ var lms = new int[m];
+ var c2 = 0;
+ for (var i = 1; i < n; ++i)
+ if (IsLms(i))
+ lms[c2++] = i;
+ for (var i = 0; i < m; ++i)
+ sa1[i] = lms[sa1[i]];
+ Array.Fill(sa, -1);
+ Buckets(true);
+ for (var i = m - 1; i >= 0; --i) {
+ var p = sa1[i];
+ sa[--bucket[s[p]]] = p;
+ }
+ Induce();
+ }
+}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqTables.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqTables.cs
new file mode 100644
index 000000000..a8122a495
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqTables.cs
@@ -0,0 +1,301 @@
+namespace FileFormat.Zpaq;
+
+/// Model-independent lookup tables of the ZPAQ context-mixing predictor.
+///
+/// Copied value-for-value from libzpaq 7.15 (Matt Mahoney, public domain); the ZPAQ
+/// specification fixes these tables, so they are reproduced exactly rather than recomputed.
+///
+internal static class ZpaqTables {
+ /// Bit-history state machine: [state*4] = next state after a 0, after a 1, then the 0 and 1 counts.
+ internal static ReadOnlySpan StateTable => [
+ 1, 2, 0, 0, 3, 5, 1, 0, 4, 6, 0, 1, 7, 9, 2, 0,
+ 8, 11, 1, 1, 8, 11, 1, 1, 10, 12, 0, 2, 13, 15, 3, 0,
+ 14, 17, 2, 1, 14, 17, 2, 1, 16, 19, 1, 2, 16, 19, 1, 2,
+ 18, 20, 0, 3, 21, 23, 4, 0, 22, 25, 3, 1, 22, 25, 3, 1,
+ 24, 27, 2, 2, 24, 27, 2, 2, 26, 29, 1, 3, 26, 29, 1, 3,
+ 28, 30, 0, 4, 31, 33, 5, 0, 32, 35, 4, 1, 32, 35, 4, 1,
+ 34, 37, 3, 2, 34, 37, 3, 2, 36, 39, 2, 3, 36, 39, 2, 3,
+ 38, 41, 1, 4, 38, 41, 1, 4, 40, 42, 0, 5, 43, 33, 6, 0,
+ 44, 47, 5, 1, 44, 47, 5, 1, 46, 49, 4, 2, 46, 49, 4, 2,
+ 48, 51, 3, 3, 48, 51, 3, 3, 50, 53, 2, 4, 50, 53, 2, 4,
+ 52, 55, 1, 5, 52, 55, 1, 5, 40, 56, 0, 6, 57, 45, 7, 0,
+ 58, 47, 6, 1, 58, 47, 6, 1, 60, 63, 5, 2, 60, 63, 5, 2,
+ 62, 65, 4, 3, 62, 65, 4, 3, 64, 67, 3, 4, 64, 67, 3, 4,
+ 66, 69, 2, 5, 66, 69, 2, 5, 52, 71, 1, 6, 52, 71, 1, 6,
+ 54, 72, 0, 7, 73, 59, 8, 0, 74, 61, 7, 1, 74, 61, 7, 1,
+ 76, 63, 6, 2, 76, 63, 6, 2, 78, 81, 5, 3, 78, 81, 5, 3,
+ 80, 83, 4, 4, 80, 83, 4, 4, 82, 85, 3, 5, 82, 85, 3, 5,
+ 66, 87, 2, 6, 66, 87, 2, 6, 68, 89, 1, 7, 68, 89, 1, 7,
+ 70, 90, 0, 8, 91, 59, 9, 0, 92, 77, 8, 1, 92, 77, 8, 1,
+ 94, 79, 7, 2, 94, 79, 7, 2, 96, 81, 6, 3, 96, 81, 6, 3,
+ 98, 101, 5, 4, 98, 101, 5, 4, 100, 103, 4, 5, 100, 103, 4, 5,
+ 82, 105, 3, 6, 82, 105, 3, 6, 84, 107, 2, 7, 84, 107, 2, 7,
+ 86, 109, 1, 8, 86, 109, 1, 8, 70, 110, 0, 9, 111, 59, 10, 0,
+ 112, 77, 9, 1, 112, 77, 9, 1, 114, 97, 8, 2, 114, 97, 8, 2,
+ 116, 99, 7, 3, 116, 99, 7, 3, 62, 101, 6, 4, 62, 101, 6, 4,
+ 80, 83, 5, 5, 80, 83, 5, 5, 100, 67, 4, 6, 100, 67, 4, 6,
+ 102, 119, 3, 7, 102, 119, 3, 7, 104, 121, 2, 8, 104, 121, 2, 8,
+ 86, 123, 1, 9, 86, 123, 1, 9, 70, 124, 0, 10, 125, 59, 11, 0,
+ 126, 77, 10, 1, 126, 77, 10, 1, 128, 97, 9, 2, 128, 97, 9, 2,
+ 60, 63, 8, 3, 60, 63, 8, 3, 66, 69, 3, 8, 66, 69, 3, 8,
+ 104, 131, 2, 9, 104, 131, 2, 9, 86, 133, 1, 10, 86, 133, 1, 10,
+ 70, 134, 0, 11, 135, 59, 12, 0, 136, 77, 11, 1, 136, 77, 11, 1,
+ 138, 97, 10, 2, 138, 97, 10, 2, 104, 141, 2, 10, 104, 141, 2, 10,
+ 86, 143, 1, 11, 86, 143, 1, 11, 70, 144, 0, 12, 145, 59, 13, 0,
+ 146, 77, 12, 1, 146, 77, 12, 1, 148, 97, 11, 2, 148, 97, 11, 2,
+ 104, 151, 2, 11, 104, 151, 2, 11, 86, 153, 1, 12, 86, 153, 1, 12,
+ 70, 154, 0, 13, 155, 59, 14, 0, 156, 77, 13, 1, 156, 77, 13, 1,
+ 158, 97, 12, 2, 158, 97, 12, 2, 104, 161, 2, 12, 104, 161, 2, 12,
+ 86, 163, 1, 13, 86, 163, 1, 13, 70, 164, 0, 14, 165, 59, 15, 0,
+ 166, 77, 14, 1, 166, 77, 14, 1, 168, 97, 13, 2, 168, 97, 13, 2,
+ 104, 171, 2, 13, 104, 171, 2, 13, 86, 173, 1, 14, 86, 173, 1, 14,
+ 70, 174, 0, 15, 175, 59, 16, 0, 176, 77, 15, 1, 176, 77, 15, 1,
+ 178, 97, 14, 2, 178, 97, 14, 2, 104, 181, 2, 14, 104, 181, 2, 14,
+ 86, 183, 1, 15, 86, 183, 1, 15, 70, 184, 0, 16, 185, 59, 17, 0,
+ 186, 77, 16, 1, 186, 77, 16, 1, 74, 97, 15, 2, 74, 97, 15, 2,
+ 104, 89, 2, 15, 104, 89, 2, 15, 86, 187, 1, 16, 86, 187, 1, 16,
+ 70, 188, 0, 17, 189, 59, 18, 0, 190, 77, 17, 1, 86, 191, 1, 17,
+ 70, 192, 0, 18, 193, 59, 19, 0, 194, 77, 18, 1, 86, 195, 1, 18,
+ 70, 196, 0, 19, 193, 59, 20, 0, 197, 77, 19, 1, 86, 198, 1, 19,
+ 70, 196, 0, 20, 199, 77, 20, 1, 86, 200, 1, 20, 201, 77, 21, 1,
+ 86, 202, 1, 21, 203, 77, 22, 1, 86, 204, 1, 22, 205, 77, 23, 1,
+ 86, 206, 1, 23, 207, 77, 24, 1, 86, 208, 1, 24, 209, 77, 25, 1,
+ 86, 210, 1, 25, 211, 77, 26, 1, 86, 212, 1, 26, 213, 77, 27, 1,
+ 86, 214, 1, 27, 215, 77, 28, 1, 86, 216, 1, 28, 217, 77, 29, 1,
+ 86, 218, 1, 29, 219, 77, 30, 1, 86, 220, 1, 30, 221, 77, 31, 1,
+ 86, 222, 1, 31, 223, 77, 32, 1, 86, 224, 1, 32, 225, 77, 33, 1,
+ 86, 226, 1, 33, 227, 77, 34, 1, 86, 228, 1, 34, 229, 77, 35, 1,
+ 86, 230, 1, 35, 231, 77, 36, 1, 86, 232, 1, 36, 233, 77, 37, 1,
+ 86, 234, 1, 37, 235, 77, 38, 1, 86, 236, 1, 38, 237, 77, 39, 1,
+ 86, 238, 1, 39, 239, 77, 40, 1, 86, 240, 1, 40, 241, 77, 41, 1,
+ 86, 242, 1, 41, 243, 77, 42, 1, 86, 244, 1, 42, 245, 77, 43, 1,
+ 86, 246, 1, 43, 247, 77, 44, 1, 86, 248, 1, 44, 249, 77, 45, 1,
+ 86, 250, 1, 45, 251, 77, 46, 1, 86, 252, 1, 46, 253, 77, 47, 1,
+ 86, 254, 1, 47, 253, 77, 48, 1, 86, 254, 1, 48, 0, 0, 0, 0,
+ ];
+
+ /// MATCH prediction divisor: 2048 / i.
+ internal static readonly int[] MatchDivisor = [
+ 0, 2048, 1024, 682, 512, 409, 341, 292, 256, 227, 204, 186, 170, 157, 146, 136,
+ 128, 120, 113, 107, 102, 97, 93, 89, 85, 81, 78, 75, 73, 70, 68, 66,
+ 64, 62, 60, 58, 56, 55, 53, 52, 51, 49, 48, 47, 46, 45, 44, 43,
+ 42, 41, 40, 40, 39, 38, 37, 37, 36, 35, 35, 34, 34, 33, 33, 32,
+ 32, 31, 31, 30, 30, 29, 29, 28, 28, 28, 27, 27, 26, 26, 26, 25,
+ 25, 25, 24, 24, 24, 24, 23, 23, 23, 23, 22, 22, 22, 22, 21, 21,
+ 21, 21, 20, 20, 20, 20, 20, 19, 19, 19, 19, 19, 18, 18, 18, 18,
+ 18, 18, 17, 17, 17, 17, 17, 17, 17, 16, 16, 16, 16, 16, 16, 16,
+ 16, 15, 15, 15, 15, 15, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12,
+ 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 10, 10, 10, 10,
+ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 9, 9, 9,
+ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+ 9, 9, 9, 9, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
+ 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
+ ];
+
+ /// CM/SSE adaptation divisor: (1 << 17) / (i * 2 + 3) * 2.
+ internal static readonly int[] CmDivisor = [
+ 87380, 52428, 37448, 29126, 23830, 20164, 17476, 15420, 13796, 12482, 11396, 10484, 9708, 9038, 8456, 7942,
+ 7488, 7084, 6720, 6392, 6096, 5824, 5576, 5348, 5140, 4946, 4766, 4598, 4442, 4296, 4160, 4032,
+ 3912, 3798, 3692, 3590, 3494, 3404, 3318, 3236, 3158, 3084, 3012, 2944, 2880, 2818, 2758, 2702,
+ 2646, 2594, 2544, 2496, 2448, 2404, 2360, 2318, 2278, 2240, 2202, 2166, 2130, 2096, 2064, 2032,
+ 2000, 1970, 1940, 1912, 1884, 1858, 1832, 1806, 1782, 1758, 1736, 1712, 1690, 1668, 1648, 1628,
+ 1608, 1588, 1568, 1550, 1532, 1514, 1496, 1480, 1464, 1448, 1432, 1416, 1400, 1386, 1372, 1358,
+ 1344, 1330, 1316, 1304, 1290, 1278, 1266, 1254, 1242, 1230, 1218, 1208, 1196, 1186, 1174, 1164,
+ 1154, 1144, 1134, 1124, 1114, 1106, 1096, 1086, 1078, 1068, 1060, 1052, 1044, 1036, 1028, 1020,
+ 1012, 1004, 996, 988, 980, 974, 966, 960, 952, 946, 938, 932, 926, 918, 912, 906,
+ 900, 894, 888, 882, 876, 870, 864, 858, 852, 848, 842, 836, 832, 826, 820, 816,
+ 810, 806, 800, 796, 790, 786, 782, 776, 772, 768, 764, 758, 754, 750, 746, 742,
+ 738, 734, 730, 726, 722, 718, 714, 710, 706, 702, 698, 694, 690, 688, 684, 680,
+ 676, 672, 670, 666, 662, 660, 656, 652, 650, 646, 644, 640, 636, 634, 630, 628,
+ 624, 622, 618, 616, 612, 610, 608, 604, 602, 598, 596, 594, 590, 588, 586, 582,
+ 580, 578, 576, 572, 570, 568, 566, 562, 560, 558, 556, 554, 550, 548, 546, 544,
+ 542, 540, 538, 536, 532, 530, 528, 526, 524, 522, 520, 518, 516, 514, 512, 510,
+ 508, 506, 504, 502, 500, 498, 496, 494, 492, 490, 488, 488, 486, 484, 482, 480,
+ 478, 476, 474, 474, 472, 470, 468, 466, 464, 462, 462, 460, 458, 456, 454, 454,
+ 452, 450, 448, 448, 446, 444, 442, 442, 440, 438, 436, 436, 434, 432, 430, 430,
+ 428, 426, 426, 424, 422, 422, 420, 418, 418, 416, 414, 414, 412, 410, 410, 408,
+ 406, 406, 404, 402, 402, 400, 400, 398, 396, 396, 394, 394, 392, 390, 390, 388,
+ 388, 386, 386, 384, 382, 382, 380, 380, 378, 378, 376, 376, 374, 372, 372, 370,
+ 370, 368, 368, 366, 366, 364, 364, 362, 362, 360, 360, 358, 358, 356, 356, 354,
+ 354, 352, 352, 350, 350, 348, 348, 348, 346, 346, 344, 344, 342, 342, 340, 340,
+ 340, 338, 338, 336, 336, 334, 334, 332, 332, 332, 330, 330, 328, 328, 328, 326,
+ 326, 324, 324, 324, 322, 322, 320, 320, 320, 318, 318, 316, 316, 316, 314, 314,
+ 312, 312, 312, 310, 310, 310, 308, 308, 308, 306, 306, 304, 304, 304, 302, 302,
+ 302, 300, 300, 300, 298, 298, 298, 296, 296, 296, 294, 294, 294, 292, 292, 292,
+ 290, 290, 290, 288, 288, 288, 286, 286, 286, 284, 284, 284, 284, 282, 282, 282,
+ 280, 280, 280, 278, 278, 278, 276, 276, 276, 276, 274, 274, 274, 272, 272, 272,
+ 272, 270, 270, 270, 268, 268, 268, 268, 266, 266, 266, 266, 264, 264, 264, 262,
+ 262, 262, 262, 260, 260, 260, 260, 258, 258, 258, 258, 256, 256, 256, 256, 254,
+ 254, 254, 254, 252, 252, 252, 252, 250, 250, 250, 250, 248, 248, 248, 248, 248,
+ 246, 246, 246, 246, 244, 244, 244, 244, 242, 242, 242, 242, 242, 240, 240, 240,
+ 240, 238, 238, 238, 238, 238, 236, 236, 236, 236, 234, 234, 234, 234, 234, 232,
+ 232, 232, 232, 232, 230, 230, 230, 230, 230, 228, 228, 228, 228, 228, 226, 226,
+ 226, 226, 226, 224, 224, 224, 224, 224, 222, 222, 222, 222, 222, 220, 220, 220,
+ 220, 220, 220, 218, 218, 218, 218, 218, 216, 216, 216, 216, 216, 216, 214, 214,
+ 214, 214, 214, 212, 212, 212, 212, 212, 212, 210, 210, 210, 210, 210, 210, 208,
+ 208, 208, 208, 208, 208, 206, 206, 206, 206, 206, 206, 204, 204, 204, 204, 204,
+ 204, 204, 202, 202, 202, 202, 202, 202, 200, 200, 200, 200, 200, 200, 198, 198,
+ 198, 198, 198, 198, 198, 196, 196, 196, 196, 196, 196, 196, 194, 194, 194, 194,
+ 194, 194, 194, 192, 192, 192, 192, 192, 192, 192, 190, 190, 190, 190, 190, 190,
+ 190, 188, 188, 188, 188, 188, 188, 188, 186, 186, 186, 186, 186, 186, 186, 186,
+ 184, 184, 184, 184, 184, 184, 184, 182, 182, 182, 182, 182, 182, 182, 182, 180,
+ 180, 180, 180, 180, 180, 180, 180, 178, 178, 178, 178, 178, 178, 178, 178, 176,
+ 176, 176, 176, 176, 176, 176, 176, 176, 174, 174, 174, 174, 174, 174, 174, 174,
+ 172, 172, 172, 172, 172, 172, 172, 172, 172, 170, 170, 170, 170, 170, 170, 170,
+ 170, 170, 168, 168, 168, 168, 168, 168, 168, 168, 168, 166, 166, 166, 166, 166,
+ 166, 166, 166, 166, 166, 164, 164, 164, 164, 164, 164, 164, 164, 164, 162, 162,
+ 162, 162, 162, 162, 162, 162, 162, 162, 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, 156, 156, 156,
+ 156, 156, 156, 156, 156, 156, 156, 154, 154, 154, 154, 154, 154, 154, 154, 154,
+ 154, 154, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, 150, 150, 150,
+ 150, 150, 150, 150, 150, 150, 150, 150, 150, 148, 148, 148, 148, 148, 148, 148,
+ 148, 148, 148, 148, 148, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 144, 144, 144, 144, 144, 144, 144, 144, 144, 144, 144, 144, 142, 142, 142,
+ 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 140, 140, 140, 140, 140, 140,
+ 140, 140, 140, 140, 140, 140, 140, 138, 138, 138, 138, 138, 138, 138, 138, 138,
+ 138, 138, 138, 138, 138, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136,
+ 136, 136, 136, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, 134,
+ 134, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 132,
+ 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 126,
+ ];
+
+ /// Middle 1344 of the 4096 squash entries, 32768 / (1 + exp((i - 2048) / -64)).
+ internal static readonly ushort[] SquashMiddle = [
+ 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3,
+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5,
+ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6,
+ 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8,
+ 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10,
+ 10, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13,
+ 14, 14, 14, 14, 15, 15, 15, 15, 15, 16, 16, 16, 17, 17, 17, 17,
+ 18, 18, 18, 18, 19, 19, 19, 20, 20, 20, 21, 21, 21, 22, 22, 22,
+ 23, 23, 23, 24, 24, 25, 25, 25, 26, 26, 27, 27, 28, 28, 28, 29,
+ 29, 30, 30, 31, 31, 32, 32, 33, 33, 34, 34, 35, 36, 36, 37, 37,
+ 38, 38, 39, 40, 40, 41, 42, 42, 43, 44, 44, 45, 46, 46, 47, 48,
+ 49, 49, 50, 51, 52, 53, 54, 54, 55, 56, 57, 58, 59, 60, 61, 62,
+ 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 76, 77, 78, 79,
+ 81, 82, 83, 84, 86, 87, 88, 90, 91, 93, 94, 96, 97, 99, 100, 102,
+ 103, 105, 107, 108, 110, 112, 114, 115, 117, 119, 121, 123, 125, 127, 129, 131,
+ 133, 135, 137, 139, 141, 144, 146, 148, 151, 153, 155, 158, 160, 163, 165, 168,
+ 171, 173, 176, 179, 182, 184, 187, 190, 193, 196, 199, 202, 206, 209, 212, 215,
+ 219, 222, 226, 229, 233, 237, 240, 244, 248, 252, 256, 260, 264, 268, 272, 276,
+ 281, 285, 289, 294, 299, 303, 308, 313, 318, 323, 328, 333, 338, 343, 349, 354,
+ 360, 365, 371, 377, 382, 388, 394, 401, 407, 413, 420, 426, 433, 440, 446, 453,
+ 460, 467, 475, 482, 490, 497, 505, 513, 521, 529, 537, 545, 554, 562, 571, 580,
+ 589, 598, 607, 617, 626, 636, 646, 656, 666, 676, 686, 697, 708, 719, 730, 741,
+ 752, 764, 776, 788, 800, 812, 825, 837, 850, 863, 876, 890, 903, 917, 931, 946,
+ 960, 975, 990, 1005, 1020, 1036, 1051, 1067, 1084, 1100, 1117, 1134, 1151, 1169, 1186, 1204,
+ 1223, 1241, 1260, 1279, 1298, 1318, 1338, 1358, 1379, 1399, 1421, 1442, 1464, 1486, 1508, 1531,
+ 1554, 1577, 1600, 1624, 1649, 1673, 1698, 1724, 1749, 1775, 1802, 1829, 1856, 1883, 1911, 1940,
+ 1968, 1998, 2027, 2057, 2087, 2118, 2149, 2181, 2213, 2245, 2278, 2312, 2345, 2380, 2414, 2450,
+ 2485, 2521, 2558, 2595, 2633, 2671, 2709, 2748, 2788, 2828, 2869, 2910, 2952, 2994, 3037, 3080,
+ 3124, 3168, 3213, 3259, 3305, 3352, 3399, 3447, 3496, 3545, 3594, 3645, 3696, 3747, 3799, 3852,
+ 3906, 3960, 4014, 4070, 4126, 4182, 4240, 4298, 4356, 4416, 4476, 4537, 4598, 4660, 4723, 4786,
+ 4851, 4916, 4981, 5048, 5115, 5183, 5251, 5320, 5390, 5461, 5533, 5605, 5678, 5752, 5826, 5901,
+ 5977, 6054, 6131, 6210, 6289, 6369, 6449, 6530, 6613, 6695, 6779, 6863, 6949, 7035, 7121, 7209,
+ 7297, 7386, 7476, 7566, 7658, 7750, 7842, 7936, 8030, 8126, 8221, 8318, 8415, 8513, 8612, 8712,
+ 8812, 8913, 9015, 9117, 9221, 9324, 9429, 9534, 9640, 9747, 9854, 9962, 10071, 10180, 10290, 10401,
+ 10512, 10624, 10737, 10850, 10963, 11078, 11192, 11308, 11424, 11540, 11658, 11775, 11893, 12012, 12131, 12251,
+ 12371, 12491, 12612, 12734, 12856, 12978, 13101, 13224, 13347, 13471, 13595, 13719, 13844, 13969, 14095, 14220,
+ 14346, 14472, 14599, 14725, 14852, 14979, 15106, 15233, 15361, 15488, 15616, 15744, 15872, 16000, 16128, 16256,
+ 16384, 16511, 16639, 16767, 16895, 17023, 17151, 17279, 17406, 17534, 17661, 17788, 17915, 18042, 18168, 18295,
+ 18421, 18547, 18672, 18798, 18923, 19048, 19172, 19296, 19420, 19543, 19666, 19789, 19911, 20033, 20155, 20276,
+ 20396, 20516, 20636, 20755, 20874, 20992, 21109, 21227, 21343, 21459, 21575, 21689, 21804, 21917, 22030, 22143,
+ 22255, 22366, 22477, 22587, 22696, 22805, 22913, 23020, 23127, 23233, 23338, 23443, 23546, 23650, 23752, 23854,
+ 23955, 24055, 24155, 24254, 24352, 24449, 24546, 24641, 24737, 24831, 24925, 25017, 25109, 25201, 25291, 25381,
+ 25470, 25558, 25646, 25732, 25818, 25904, 25988, 26072, 26154, 26237, 26318, 26398, 26478, 26557, 26636, 26713,
+ 26790, 26866, 26941, 27015, 27089, 27162, 27234, 27306, 27377, 27447, 27516, 27584, 27652, 27719, 27786, 27851,
+ 27916, 27981, 28044, 28107, 28169, 28230, 28291, 28351, 28411, 28469, 28527, 28585, 28641, 28697, 28753, 28807,
+ 28861, 28915, 28968, 29020, 29071, 29122, 29173, 29222, 29271, 29320, 29368, 29415, 29462, 29508, 29554, 29599,
+ 29643, 29687, 29730, 29773, 29815, 29857, 29898, 29939, 29979, 30019, 30058, 30096, 30134, 30172, 30209, 30246,
+ 30282, 30317, 30353, 30387, 30422, 30455, 30489, 30522, 30554, 30586, 30618, 30649, 30680, 30710, 30740, 30769,
+ 30799, 30827, 30856, 30884, 30911, 30938, 30965, 30992, 31018, 31043, 31069, 31094, 31118, 31143, 31167, 31190,
+ 31213, 31236, 31259, 31281, 31303, 31325, 31346, 31368, 31388, 31409, 31429, 31449, 31469, 31488, 31507, 31526,
+ 31544, 31563, 31581, 31598, 31616, 31633, 31650, 31667, 31683, 31700, 31716, 31731, 31747, 31762, 31777, 31792,
+ 31807, 31821, 31836, 31850, 31864, 31877, 31891, 31904, 31917, 31930, 31942, 31955, 31967, 31979, 31991, 32003,
+ 32015, 32026, 32037, 32048, 32059, 32070, 32081, 32091, 32101, 32111, 32121, 32131, 32141, 32150, 32160, 32169,
+ 32178, 32187, 32196, 32205, 32213, 32222, 32230, 32238, 32246, 32254, 32262, 32270, 32277, 32285, 32292, 32300,
+ 32307, 32314, 32321, 32327, 32334, 32341, 32347, 32354, 32360, 32366, 32373, 32379, 32385, 32390, 32396, 32402,
+ 32407, 32413, 32418, 32424, 32429, 32434, 32439, 32444, 32449, 32454, 32459, 32464, 32468, 32473, 32478, 32482,
+ 32486, 32491, 32495, 32499, 32503, 32507, 32511, 32515, 32519, 32523, 32527, 32530, 32534, 32538, 32541, 32545,
+ 32548, 32552, 32555, 32558, 32561, 32565, 32568, 32571, 32574, 32577, 32580, 32583, 32585, 32588, 32591, 32594,
+ 32596, 32599, 32602, 32604, 32607, 32609, 32612, 32614, 32616, 32619, 32621, 32623, 32626, 32628, 32630, 32632,
+ 32634, 32636, 32638, 32640, 32642, 32644, 32646, 32648, 32650, 32652, 32653, 32655, 32657, 32659, 32660, 32662,
+ 32664, 32665, 32667, 32668, 32670, 32671, 32673, 32674, 32676, 32677, 32679, 32680, 32681, 32683, 32684, 32685,
+ 32686, 32688, 32689, 32690, 32691, 32693, 32694, 32695, 32696, 32697, 32698, 32699, 32700, 32701, 32702, 32703,
+ 32704, 32705, 32706, 32707, 32708, 32709, 32710, 32711, 32712, 32713, 32713, 32714, 32715, 32716, 32717, 32718,
+ 32718, 32719, 32720, 32721, 32721, 32722, 32723, 32723, 32724, 32725, 32725, 32726, 32727, 32727, 32728, 32729,
+ 32729, 32730, 32730, 32731, 32731, 32732, 32733, 32733, 32734, 32734, 32735, 32735, 32736, 32736, 32737, 32737,
+ 32738, 32738, 32739, 32739, 32739, 32740, 32740, 32741, 32741, 32742, 32742, 32742, 32743, 32743, 32744, 32744,
+ 32744, 32745, 32745, 32745, 32746, 32746, 32746, 32747, 32747, 32747, 32748, 32748, 32748, 32749, 32749, 32749,
+ 32749, 32750, 32750, 32750, 32750, 32751, 32751, 32751, 32752, 32752, 32752, 32752, 32752, 32753, 32753, 32753,
+ 32753, 32754, 32754, 32754, 32754, 32754, 32755, 32755, 32755, 32755, 32755, 32756, 32756, 32756, 32756, 32756,
+ 32757, 32757, 32757, 32757, 32757, 32757, 32757, 32758, 32758, 32758, 32758, 32758, 32758, 32759, 32759, 32759,
+ 32759, 32759, 32759, 32759, 32759, 32760, 32760, 32760, 32760, 32760, 32760, 32760, 32760, 32761, 32761, 32761,
+ 32761, 32761, 32761, 32761, 32761, 32761, 32761, 32762, 32762, 32762, 32762, 32762, 32762, 32762, 32762, 32762,
+ 32762, 32762, 32762, 32763, 32763, 32763, 32763, 32763, 32763, 32763, 32763, 32763, 32763, 32763, 32763, 32763,
+ 32763, 32764, 32764, 32764, 32764, 32764, 32764, 32764, 32764, 32764, 32764, 32764, 32764, 32764, 32764, 32764,
+ 32764, 32764, 32764, 32764, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765,
+ 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32765, 32766, 32766,
+ 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766,
+ 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766,
+ 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32766, 32767, 32767, 32767, 32767, 32767, 32767,
+ ];
+
+ /// Run lengths that expand into the upper half of the stretch table.
+ internal static ReadOnlySpan StretchRunLengths => [
+ 64, 128, 128, 128, 128, 128, 127, 128, 127, 128, 127, 127, 127, 127, 126, 126,
+ 126, 126, 126, 125, 125, 124, 125, 124, 123, 123, 123, 123, 122, 122, 121, 121,
+ 120, 120, 119, 119, 118, 118, 118, 116, 117, 115, 116, 114, 114, 113, 113, 112,
+ 112, 111, 110, 110, 109, 108, 108, 107, 106, 106, 105, 104, 104, 102, 103, 101,
+ 101, 100, 99, 98, 98, 97, 96, 96, 94, 94, 94, 92, 92, 91, 90, 89,
+ 89, 88, 87, 86, 86, 84, 84, 84, 82, 82, 81, 80, 79, 79, 78, 77,
+ 76, 76, 75, 74, 73, 73, 72, 71, 70, 70, 69, 68, 67, 67, 66, 65,
+ 65, 64, 63, 62, 62, 61, 61, 59, 59, 59, 57, 58, 56, 56, 55, 54,
+ 54, 53, 52, 52, 51, 51, 50, 49, 49, 48, 48, 47, 47, 45, 46, 44,
+ 45, 43, 43, 43, 42, 41, 41, 40, 40, 40, 39, 38, 38, 37, 37, 36,
+ 36, 36, 35, 34, 34, 34, 33, 32, 33, 32, 31, 31, 30, 31, 29, 30,
+ 28, 29, 28, 28, 27, 27, 27, 26, 26, 25, 26, 24, 25, 24, 24, 23,
+ 23, 23, 23, 22, 22, 21, 22, 21, 20, 21, 20, 19, 20, 19, 19, 19,
+ 18, 18, 18, 18, 17, 17, 17, 17, 16, 16, 16, 16, 15, 15, 15, 15,
+ 15, 14, 14, 14, 14, 13, 14, 13, 13, 13, 12, 13, 12, 12, 12, 11,
+ 12, 11, 11, 11, 11, 11, 10, 11, 10, 10, 10, 10, 9, 10, 9, 9,
+ 9, 9, 9, 8, 9, 8, 9, 8, 8, 8, 7, 8, 8, 7, 7, 8,
+ 7, 7, 7, 6, 7, 7, 6, 6, 7, 6, 6, 6, 6, 6, 6, 5,
+ 6, 5, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 5, 4, 5,
+ 4, 4, 5, 4, 4, 4, 4, 4, 4, 3, 4, 4, 3, 4, 4, 3,
+ 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3,
+ 2, 3, 2, 3, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 2, 3,
+ 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 1, 2, 2, 2, 1,
+ 2, 1, 2, 2, 1, 2, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1,
+ 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0,
+ 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0,
+ 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1,
+ 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1,
+ 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0,
+ 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0,
+ 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
+ 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 1, 0,
+ ];
+}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqWriter.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqWriter.cs
index 301887751..f0bda5834 100644
--- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqWriter.cs
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqWriter.cs
@@ -1,246 +1,384 @@
using System.Text;
-using Compression.Core.Checksums;
namespace FileFormat.Zpaq;
///
-/// Writes a ZPAQ level-1 journaling archive.
+/// Writes a ZPAQ journaling archive holding one version (transaction): the c header,
+/// the deduplicated and compressed d data blocks, their h fragment tables and the
+/// i index, exactly as zpaq add lays them out.
///
///
///
-/// Each call to or creates a
-/// transaction consisting of a header block ('c'), a data block ('d'), and an
-/// index block ('h'). Data is stored uncompressed (no ZPAQL program is needed).
+/// Follows Jidac::add of zpaq 7.15 (Matt Mahoney, public domain): files are split into
+/// content-defined fragments with the order-1 rolling hash, identical fragments are stored
+/// once, fragments are packed into blocks (16 MiB at method 1, 64 MiB at 2..5) that end with
+/// the fragment-size list, and each block's method is chosen from the analysed redundancy
+/// and text/x86 hints. The whole archive is produced when the writer is disposed.
///
///
-/// The writer produces archives that can be read by the reference zpaq
-/// tool (level 1 journaling format) and by .
+/// accepts zpaq's -method values: 0..5 with an
+/// optional block-size digit ("14", "36"), or an explicit x method string.
///
///
public sealed class ZpaqWriter : IDisposable {
+ private const int OrderOneTables = 4;
+ private const int Fragment = 6;
+
+ // 32768/((i+1)*204), zpaq 7.15 Jidac::add(); entries past the 160 listed are zero.
+ private static readonly byte[] AnalysisDivisor = BuildAnalysisDivisor();
+
private readonly Stream _stream;
private readonly bool _leaveOpen;
+ private readonly Dictionary _entries = new(StringComparer.Ordinal);
private bool _disposed;
- // Accumulate SHA-1 hashes for the final index block.
- private readonly List _hashes = [];
- // Track whether we have written anything (for the final index block).
- private bool _hasEntries;
-
- ///
- /// Initializes a new ZPAQ writer that writes to the specified stream.
- ///
- /// A writable stream to receive the archive data.
- ///
- /// to leave open when
- /// this writer is disposed; (default) to dispose it.
- ///
- ///
- /// Thrown when is .
- ///
- ///
- /// Thrown when is not writable.
- ///
- public ZpaqWriter(Stream stream, bool leaveOpen = false) {
+ private sealed class PendingEntry(string name, long date, long attr, byte[]? data) {
+ public readonly string Name = name;
+ public readonly long Date = date;
+ public readonly long Attr = attr;
+ public readonly byte[]? Data = data;
+ public readonly List Ptr = [];
+ public ulong SortKey;
+ }
+
+ /// Creates a writer that emits the archive to on dispose.
+ /// Writable destination.
+ /// Whether to leave open on dispose.
+ /// zpaq compression method; "1" is zpaq's default.
+ public ZpaqWriter(Stream stream, bool leaveOpen = false, string method = "1") {
ArgumentNullException.ThrowIfNull(stream);
+ ArgumentException.ThrowIfNullOrEmpty(method);
if (!stream.CanWrite)
throw new ArgumentException("Stream must be writable.", nameof(stream));
-
- _stream = stream;
- _leaveOpen = leaveOpen;
+ if (!"0123456789x".Contains(method[0]))
+ throw new ArgumentException("Method must begin with 0..5 or x.", nameof(method));
+ this._stream = stream;
+ this._leaveOpen = leaveOpen;
+ this.Method = method;
}
- ///
- /// Adds a file entry to the archive with the given name and data.
- ///
- ///
- /// The filename to store in the archive. Forward slashes are used as path separators.
- ///
- /// The uncompressed file data.
- ///
- /// The last-modified timestamp for the file.
- /// Defaults to if not specified.
- ///
- /// Thrown if the writer has been disposed.
- ///
- /// Thrown when or is .
- ///
+ /// The zpaq compression method used for the data blocks.
+ public string Method { get; }
+
+ /// Date of the transaction; defaults to the time of disposal.
+ public DateTime? TransactionDate { get; set; }
+
+ /// Adds (or replaces) a file.
public void AddFile(string fileName, byte[] data, DateTime? lastModified = null) {
- ObjectDisposedException.ThrowIf(_disposed, this);
+ ObjectDisposedException.ThrowIf(this._disposed, this);
ArgumentNullException.ThrowIfNull(fileName);
ArgumentNullException.ThrowIfNull(data);
-
- var timestamp = lastModified ?? DateTime.UtcNow;
- var normalizedName = NormalizeName(fileName);
-
- // Compute SHA-1 hash of the data.
- var hash = Sha1.Compute(data);
-
- // Write header block ('c').
- WriteHeaderBlock(normalizedName, (long)data.Length, timestamp, isDirectory: false);
-
- // Write data block ('d').
- WriteDataBlock(data);
-
- // Record hash for the index block.
- _hashes.Add(hash);
- _hasEntries = true;
+ var name = fileName.Replace('\\', '/').TrimStart('/');
+ ArgumentException.ThrowIfNullOrEmpty(name, nameof(fileName));
+ if (name.EndsWith('/'))
+ throw new ArgumentException("A file name cannot end with '/'.", nameof(fileName));
+ this._entries[name] = new(name, this.DateOf(lastModified), 'w' | 0x20L << 8, data);
}
- ///
- /// Adds a directory entry to the archive.
- ///
- ///
- /// The directory name. A trailing '/' is added if not already present.
- ///
- ///
- /// The last-modified timestamp. Defaults to if not specified.
- ///
- /// Thrown if the writer has been disposed.
- ///
- /// Thrown when is .
- ///
+ /// Adds a directory entry.
public void AddDirectory(string dirName, DateTime? lastModified = null) {
- ObjectDisposedException.ThrowIf(_disposed, this);
+ ObjectDisposedException.ThrowIf(this._disposed, this);
ArgumentNullException.ThrowIfNull(dirName);
-
- var timestamp = lastModified ?? DateTime.UtcNow;
- var normalizedName = NormalizeName(dirName);
- if (!normalizedName.EndsWith('/'))
- normalizedName += '/';
-
- // Write header block ('c') with size 0.
- WriteHeaderBlock(normalizedName, 0, timestamp, isDirectory: true);
-
- // Write an empty data block ('d').
- WriteDataBlock([]);
-
- // Directory entries get a zero hash.
- _hashes.Add(new byte[Sha1.HashSize]);
- _hasEntries = true;
+ var name = dirName.Replace('\\', '/').TrimStart('/');
+ ArgumentException.ThrowIfNullOrEmpty(name, nameof(dirName));
+ if (!name.EndsWith('/'))
+ name += '/';
+ this._entries[name] = new(name, this.DateOf(lastModified), 'w' | 0x10L << 8, null);
}
- ///
+ ///
public void Dispose() {
- if (_disposed)
+ if (this._disposed)
return;
+ this._disposed = true;
+ try {
+ if (this._entries.Count > 0)
+ this.WriteTransaction();
+ this._stream.Flush();
+ } finally {
+ if (!this._leaveOpen)
+ this._stream.Dispose();
+ }
+ }
- _disposed = true;
-
- // Write the final index ('h') block with all accumulated SHA-1 hashes.
- if (_hasEntries)
- WriteIndexBlock();
+ private long DateOf(DateTime? time) => ZpaqDate.FromDateTime(time ?? this.TransactionDate ?? DateTime.UtcNow);
+
+ private void WriteTransaction() {
+ var date = ZpaqDate.FromDateTime(this.TransactionDate ?? DateTime.UtcNow);
+ if (date is < 19000000000000L or >= 30000000000000L)
+ throw new InvalidOperationException("ZPAQ dates must lie in the years 1900..2999.");
+
+ var method = this.Method;
+ if (method.Length == 1)
+ method += method[0] is >= '2' and <= '9' ? "6" : "4";
+ var logBlockSize = 20 + AtoiPrefix(method, 1);
+ if (logBlockSize is < 20 or > 31)
+ throw new InvalidOperationException("ZPAQ block size must be 0..11.");
+ var blocksize = (uint)((1L << logBlockSize) - 4096);
+ var maxFragment = (8128u << Fragment) > blocksize - 12 ? blocksize - 12 : 8128u << Fragment;
+ var minFragment = (64u << Fragment) > maxFragment ? maxFragment : 64u << Fragment;
+
+ // Files to compress, sorted like zpaq: by extension (first 5 bytes), then descending size.
+ var vf = new List();
+ foreach (var e in this._entries.Values) {
+ if (e.Data == null)
+ continue;
+ ulong key = 0;
+ var sp = 0;
+ foreach (var b in Encoding.UTF8.GetBytes(e.Name)) {
+ ulong c = b;
+ if (c is >= 'A' and <= 'Z')
+ c += 'a' - 'A';
+ if (c == '/') {
+ sp = 0;
+ key = 0;
+ } else if (c == '.') {
+ sp = 8;
+ key = 0;
+ } else if (sp > 3)
+ key += c << (--sp * 8);
+ }
+ var s = (long)e.Data.Length >> 14;
+ if (s >= 1 << 24)
+ s = (1 << 24) - 1;
+ key += (ulong)((1 << 24) - s - 1);
+ e.SortKey = key;
+ vf.Add(e);
+ }
+ vf.Sort((a, b) => a.SortKey != b.SortKey ? a.SortKey.CompareTo(b.SortKey) : CompareUtf8(a.Name, b.Name));
+
+ // Fragment table: index 0 is unused, IDs start at 1.
+ var ht = new List<(byte[] Sha1, int Usize)> { ([], -2) };
+ var htIndex = new Dictionary(StringComparer.Ordinal);
+ const uint htsize = 1;
+
+ var dataBlocks = new MemoryStream();
+ var csize = new List();
+ var blocklist = new List();
+ var sb = new ZpaqBuffer();
+ uint frags = 0, redundancy = 0, text = 0, exe = 0;
+ var o1prev = new byte[OrderOneTables * 256];
+ var fragbuf = new byte[maxFragment];
+ var dataWriter = new ZpaqStreamWriter(dataBlocks);
+
+ for (var fi = 0; fi <= vf.Count; ++fi) {
+ var data = fi < vf.Count ? vf[fi].Data! : null;
+ var pos = 0;
+ for (var fj = 0; ; ++fj) {
+ long sz = 0;
+ uint hits = 0;
+ var c = -1;
+ uint htptr = 0;
+ byte[] sha1Result = [];
+ var o1 = new byte[256];
+ if (data != null) {
+ var c1 = 0;
+ var h = 0u;
+ while (true) {
+ c = pos < data.Length ? data[pos++] : -1;
+ if (c != -1) {
+ if (c == o1[c1]) {
+ h = (h + (uint)c + 1) * 314159265u;
+ ++hits;
+ } else
+ h = (h + (uint)c + 1) * 271828182u;
+ o1[c1] = (byte)c;
+ c1 = c;
+ fragbuf[sz++] = (byte)c;
+ }
+ if (c == -1 || sz >= maxFragment || (h < 1u << (22 - Fragment) && sz >= minFragment))
+ break;
+ }
+ sha1Result = Compression.Core.Checksums.Sha1.Compute(fragbuf.AsSpan(0, (int)sz));
+ htptr = htIndex.GetValueOrDefault(Convert.ToHexString(sha1Result));
+ }
+
+ if (htptr == 0) {
+ var text1 = 0;
+ var exe1 = 0;
+ var h1 = sz;
+ var o1ct = new byte[256];
+ for (var i = 0; i < 256; ++i) {
+ if (o1ct[o1[i]] < 255)
+ h1 -= (sz * AnalysisDivisor[o1ct[o1[i]]++]) >> 15;
+ if (o1[i] == ' ' && (char.IsAsciiLetterOrDigit((char)i) || i == '.' || i == ','))
+ ++text1;
+ if (o1[i] != 0 && (i < 9 || i == 11 || i == 12 || (i >= 14 && i <= 31) || i >= 240))
+ --text1;
+ if (i is >= 192 and < 240 && o1[i] != 0 && (o1[i] < 128 || o1[i] >= 192))
+ --text1;
+ if (o1[i] == 139)
+ ++exe1;
+ }
+ text1 = text1 >= 3 ? 1 : 0;
+ exe1 = exe1 >= 5 ? 1 : 0;
+ if (sz > 0)
+ h1 = h1 * h1 / sz;
+ var h2 = (uint)h1;
+ if (h2 > hits)
+ hits = h2;
+ h2 = (uint)(o1ct[0] * sz / 256);
+ if (h2 > hits)
+ hits = h2;
+ h2 = 0;
+ for (var i = 0; i < 256 * OrderOneTables; ++i)
+ if (o1prev[i] == o1[i & 255])
+ ++h2;
+ h2 = (uint)(h2 * sz / (256 * OrderOneTables));
+ if (h2 > hits)
+ hits = h2;
+ if (hits > sz)
+ hits = (uint)sz;
+
+ var newblock = false;
+ if (frags > 0 && fj == 0 && fi < vf.Count) {
+ long esize = vf[fi].Data!.Length;
+ var newsize = sb.Size + esize + (esize >> 14) + 4096 + frags * 4L;
+ if (newsize > blocksize / 4 && redundancy < sb.Size / 128)
+ newblock = true;
+ if (newblock) {
+ var ct = 0u;
+ for (var i = 0; i < 256 * OrderOneTables; ++i)
+ if (o1prev[i] != 0 && o1prev[i] == o1[i & 255])
+ ++ct;
+ if (ct > OrderOneTables * 2)
+ newblock = false;
+ }
+ if (newsize >= blocksize)
+ newblock = true;
+ }
+ if (sb.Size + sz + 80 + frags * 4L >= blocksize)
+ newblock = true;
+ if (fi == vf.Count)
+ newblock = true;
+ if (frags < 1)
+ newblock = false;
+
+ if (newblock) {
+ for (var i = ht.Count - (int)frags; i < ht.Count; ++i)
+ PutLe(sb, (ulong)ht[i].Usize, 4);
+ PutLe(sb, 0, 4);
+ PutLe(sb, frags, 4);
+ var m = method;
+ if (char.IsAsciiDigit(method[0]))
+ m += "," + ZpaqMethods.Itos(redundancy / (sb.Size / 256 + 1)) + "," + ZpaqMethods.Itos((exe > frags ? 2 : 0) + (text > frags ? 1 : 0));
+ var fn = "jDC" + ZpaqMethods.Itos(date, 14) + "d" + ZpaqMethods.Itos(ht.Count - frags, 10);
+ var before = dataBlocks.Length;
+ ZpaqMethods.CompressBlock(sb, dataWriter, m, fn, [.. JournalComment]);
+ csize.Add((int)(dataBlocks.Length - before));
+ sb = new();
+ blocklist.Add(ht.Count - (int)frags);
+ frags = redundancy = text = exe = 0;
+ Array.Clear(o1prev);
+ }
+
+ sb.Write(fragbuf.AsSpan(0, (int)sz));
+ ++frags;
+ redundancy += hits;
+ exe += (uint)exe1 * 4;
+ text += (uint)text1 * 2;
+ if (sz >= minFragment) {
+ Array.Copy(o1prev, 256, o1prev, 0, 256 * (OrderOneTables - 1));
+ Array.Copy(o1, 0, o1prev, 256 * (OrderOneTables - 1), 256);
+ }
+ }
+
+ if (data != null) {
+ if (htptr == 0) {
+ htptr = (uint)ht.Count;
+ ht.Add((sha1Result, (int)sz));
+ htIndex.TryAdd(Convert.ToHexString(sha1Result), htptr);
+ }
+ vf[fi].Ptr.Add(htptr);
+ }
+ if (c == -1)
+ break;
+ }
+ }
- _stream.Flush();
+ var output = new ZpaqStreamWriter(this._stream);
+
+ // Transaction header: the size of the data blocks that follow it.
+ var header = new ZpaqBuffer();
+ PutLe(header, (ulong)dataBlocks.Length, 8);
+ ZpaqMethods.CompressBlock(header, output, "0", "jDC" + ZpaqMethods.Itos(date, 14) + "c" + ZpaqMethods.Itos(htsize, 10), [.. JournalComment]);
+
+ dataBlocks.Position = 0;
+ dataBlocks.CopyTo(this._stream);
+
+ // Fragment tables, one per data block.
+ blocklist.Add(ht.Count);
+ for (var i = 0; i < csize.Count; ++i) {
+ if (blocklist[i] >= blocklist[i + 1])
+ continue;
+ var table = new ZpaqBuffer();
+ PutLe(table, (ulong)csize[i], 4);
+ for (var j = blocklist[i]; j < blocklist[i + 1]; ++j) {
+ table.Write(ht[j].Sha1);
+ PutLe(table, (ulong)ht[j].Usize, 4);
+ }
+ ZpaqMethods.CompressBlock(table, output, "0", "jDC" + ZpaqMethods.Itos(date, 14) + "h" + ZpaqMethods.Itos(blocklist[i], 10), [.. JournalComment]);
+ }
- if (!_leaveOpen)
- _stream.Dispose();
+ // Index, in byte order of the names like zpaq's std::map.
+ var index = new ZpaqBuffer();
+ var dtcount = 0;
+ var ordered = this._entries.Values.OrderBy(e => e.Name, Comparer.Create(CompareUtf8)).ToList();
+ for (var k = 0; ; ++k) {
+ if (k < ordered.Count) {
+ var e = ordered[k];
+ PutLe(index, (ulong)e.Date, 8);
+ index.Write(Encoding.UTF8.GetBytes(e.Name));
+ index.Put(0);
+ PutLe(index, 5, 4);
+ PutLe(index, (ulong)e.Attr, 5);
+ PutLe(index, (ulong)e.Ptr.Count, 4);
+ foreach (var p in e.Ptr)
+ PutLe(index, p, 4);
+ }
+ if (index.Size > 16000 || (index.Size > 0 && k >= ordered.Count)) {
+ ZpaqMethods.CompressBlock(index, output, "1", "jDC" + ZpaqMethods.Itos(date) + "i" + ZpaqMethods.Itos(++dtcount, 10), [.. JournalComment]);
+ index = new();
+ }
+ if (k >= ordered.Count)
+ break;
+ }
}
- // ── Block writers ──────────────────────────────────────────────────────────
-
- ///
- /// Writes a header ('c') block for a single file or directory entry.
- ///
- ///
- /// Layout:
- ///
- /// "zPQ" (3 bytes) + level (1) + 'c' (1)
- /// FILETIME (8 bytes, little-endian Windows FILETIME)
- /// attribute (1 byte: 0x20 = file, 0x10 = directory)
- /// filename (UTF-8, null-terminated)
- /// size (8 bytes, little-endian int64)
- /// 0xFF (end-of-block marker)
- ///
- ///
- private void WriteHeaderBlock(string name, long size, DateTime timestamp, bool isDirectory) {
- // Block prefix.
- _stream.Write(ZpaqConstants.BlockPrefix);
- _stream.WriteByte(ZpaqConstants.Level1);
- _stream.WriteByte(ZpaqConstants.BlockTypeHeader);
-
- // Windows FILETIME (100-nanosecond intervals since 1601-01-01 UTC).
- var fileTime = EncodeWindowsFileTime(timestamp);
- Span ftBytes = stackalloc byte[8];
- BitConverter.TryWriteBytes(ftBytes, fileTime);
- _stream.Write(ftBytes);
-
- // Attribute byte: 0x20 = normal file, 0x10 = directory (Windows convention).
- _stream.WriteByte(isDirectory ? (byte)0x10 : (byte)0x20);
-
- // Null-terminated UTF-8 filename.
- var nameBytes = Encoding.UTF8.GetBytes(name);
- _stream.Write(nameBytes);
- _stream.WriteByte(0); // null terminator
-
- // Uncompressed size (8 bytes, little-endian).
- Span sizeBytes = stackalloc byte[8];
- BitConverter.TryWriteBytes(sizeBytes, size);
- _stream.Write(sizeBytes);
-
- // End-of-block marker.
- _stream.WriteByte(0xFF);
- }
+ private static ReadOnlySpan JournalComment => "jDC\x01"u8;
- ///
- /// Writes a data ('d') block containing the raw (uncompressed) file data.
- ///
- ///
- /// Layout:
- ///
- /// "zPQ" (3 bytes) + level (1) + 'd' (1)
- /// raw data bytes
- ///
- ///
- private void WriteDataBlock(byte[] data) {
- // Block prefix.
- _stream.Write(ZpaqConstants.BlockPrefix);
- _stream.WriteByte(ZpaqConstants.Level1);
- _stream.WriteByte(ZpaqConstants.BlockTypeData);
-
- // Raw data payload.
- if (data.Length > 0)
- _stream.Write(data);
+ private static void PutLe(ZpaqBuffer sb, ulong x, int n) {
+ for (; n > 0; --n) {
+ sb.Put((int)(x & 255));
+ x >>= 8;
+ }
}
- ///
- /// Writes the index ('h') block containing SHA-1 hashes for all files
- /// added to the archive.
- ///
- ///
- /// Layout:
- ///
- /// "zPQ" (3 bytes) + level (1) + 'h' (1)
- /// For each file: SHA-1 hash (20 bytes)
- ///
- ///
- private void WriteIndexBlock() {
- // Block prefix.
- _stream.Write(ZpaqConstants.BlockPrefix);
- _stream.WriteByte(ZpaqConstants.Level1);
- _stream.WriteByte(ZpaqConstants.BlockTypeIndex);
-
- // SHA-1 hashes, one per entry.
- foreach (var hash in _hashes)
- _stream.Write(hash);
+ private static int AtoiPrefix(string s, int i) {
+ var r = 0;
+ for (; i < s.Length && char.IsAsciiDigit(s[i]); ++i)
+ r = r * 10 + s[i] - '0';
+ return r;
}
- // ── Utilities ──────────────────────────────────────────────────────────────
-
- private static string NormalizeName(string name) =>
- name.Replace('\\', '/');
-
- ///
- /// Encodes a as a Windows FILETIME (little-endian int64).
- ///
- private static long EncodeWindowsFileTime(DateTime dateTime) {
- try {
- return dateTime.Kind == DateTimeKind.Utc
- ? dateTime.ToFileTimeUtc()
- : dateTime.ToUniversalTime().ToFileTimeUtc();
- } catch {
- // If the date is out of range for FILETIME, return 0.
- return 0;
- }
+ private static int CompareUtf8(string? a, string? b) =>
+ Encoding.UTF8.GetBytes(a ?? "").AsSpan().SequenceCompareTo(Encoding.UTF8.GetBytes(b ?? ""));
+
+ private static byte[] BuildAnalysisDivisor() {
+ byte[] listed = [
+ 160, 80, 53, 40, 32, 26, 22, 20, 17, 16, 14, 13, 12, 11, 10, 10,
+ 9, 8, 8, 8, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5,
+ 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3,
+ 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ ];
+ var t = new byte[256];
+ listed.CopyTo(t, 0);
+ return t;
}
}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqlMachine.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqlMachine.cs
new file mode 100644
index 000000000..a0cc3b412
--- /dev/null
+++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqlMachine.cs
@@ -0,0 +1,462 @@
+namespace FileFormat.Zpaq;
+
+///
+/// The ZPAQL virtual machine: a block header (COMP + HCOMP, or a PCOMP post-processor) and
+/// the interpreter that runs it once per byte.
+///
+///
+/// Converted from libzpaq 7.15 (Matt Mahoney, public domain), class ZPAQL, interpreter
+/// only (no JIT). The header layout is the one in the ZPAQ Level 2 specification section 6:
+/// hsize[2] hh hm ph pm n COMP[...] 0 (128-byte guard) HCOMP[...] 0.
+///
+internal sealed class ZpaqlMachine {
+ /// Number of bytes that each component type occupies in COMP, indexed by type.
+ internal static ReadOnlySpan ComponentSize => [0, 2, 3, 2, 3, 4, 6, 6, 3, 5];
+
+ /// Component type codes.
+ internal const int None = 0, Cons = 1, Cm = 2, Icm = 3, Match = 4, Avg = 5, Mix2 = 6, Mix = 7, Isse = 8, Sse = 9;
+
+ private const int OutputBufferSize = 1 << 14;
+
+ private byte[] _m = [];
+ private uint[] _h = [];
+ private uint[] _r = [];
+ private int _mMask;
+ private int _hMask;
+ private readonly byte[] _outbuf = new byte[OutputBufferSize];
+ private int _bufptr;
+ private uint _a, _b, _c, _d;
+ private bool _f;
+ private int _pc;
+
+ /// Block header: hsize[2] hh hm ph pm n COMP (guard) HCOMP (guard).
+ internal byte[] Header = [];
+
+ /// End of COMP in : COMP occupies Header[7..Cend-1].
+ internal int Cend;
+
+ /// Start of HCOMP/PCOMP code in .
+ internal int Hbegin;
+
+ /// End of HCOMP/PCOMP code in .
+ internal int Hend;
+
+ /// Destination of the OUT instruction, or to discard.
+ internal IZpaqWriter? Output;
+
+ /// Hash of everything written by OUT, or .
+ internal ZpaqSha1? Sha1;
+
+ /// Creates an empty machine.
+ public ZpaqlMachine() => this.Clear();
+
+ /// Gets element of H (index wraps to the array size).
+ internal uint H(int i) => this._h[i & this._hMask];
+
+ /// Releases the program and resets the machine state.
+ internal void Clear() {
+ this.Cend = this.Hbegin = this.Hend = 0;
+ this._a = this._b = this._c = this._d = 0;
+ this._f = false;
+ this._pc = 0;
+ this.Header = [];
+ this._h = [];
+ this._m = [];
+ this._r = [];
+ }
+
+ /// Writes the header; with only the PCOMP size and code.
+ /// when there was a program to write.
+ internal bool Write(IZpaqWriter output, bool postProcessorOnly) {
+ if (this.Header.Length <= 6)
+ return false;
+ if (!postProcessorOnly) {
+ for (var i = 0; i < this.Cend; ++i)
+ output.Put(this.Header[i]);
+ } else {
+ output.Put((this.Hend - this.Hbegin) & 255);
+ output.Put((this.Hend - this.Hbegin) >> 8);
+ }
+ for (var i = this.Hbegin; i < this.Hend; ++i)
+ output.Put(this.Header[i]);
+ return true;
+ }
+
+ /// Reads a COMP + HCOMP header from .
+ /// The number of header bytes held.
+ internal int Read(IZpaqReader input) {
+ var hsize = input.Get();
+ hsize += input.Get() * 256;
+ if (hsize < 0)
+ ZpaqError.Throw("unexpected end of file");
+ this.Header = new byte[hsize + 300];
+ this.Cend = this.Hbegin = this.Hend = 0;
+ this.Header[this.Cend++] = (byte)hsize;
+ this.Header[this.Cend++] = (byte)(hsize >> 8);
+ while (this.Cend < 7)
+ this.Header[this.Cend++] = (byte)input.Get();
+
+ var n = this.Header[this.Cend - 1];
+ for (var i = 0; i < n; ++i) {
+ var type = input.Get();
+ if (type is < 0 or > 255)
+ ZpaqError.Throw("unexpected end of file");
+ this.Header[this.Cend++] = (byte)type;
+ var size = type < ComponentSize.Length ? ComponentSize[type] : 0;
+ if (size < 1)
+ ZpaqError.Throw("invalid component type");
+ if (this.Cend + size > hsize)
+ ZpaqError.Throw("COMP overflows header");
+ for (var j = 1; j < size; ++j)
+ this.Header[this.Cend++] = (byte)input.Get();
+ }
+ if ((this.Header[this.Cend++] = (byte)input.Get()) != 0)
+ ZpaqError.Throw("missing COMP END");
+
+ this.Hbegin = this.Hend = this.Cend + 128;
+ if (this.Hend > hsize + 129)
+ ZpaqError.Throw("missing HCOMP");
+ while (this.Hend < hsize + 129) {
+ var op = input.Get();
+ if (op == -1)
+ ZpaqError.Throw("unexpected end of file");
+ this.Header[this.Hend++] = (byte)op;
+ }
+ if ((this.Header[this.Hend++] = (byte)input.Get()) != 0)
+ ZpaqError.Throw("missing HCOMP END");
+ return this.Cend + this.Hend - this.Hbegin;
+ }
+
+ /// Prepares the machine to run as HCOMP (sizes hh, hm).
+ internal void InitHcomp() => this.Init(this.Header[2], this.Header[3]);
+
+ /// Prepares the machine to run as PCOMP (sizes ph, pm).
+ internal void InitPcomp() => this.Init(this.Header[4], this.Header[5]);
+
+ /// Approximate memory needed to run this block, in bytes.
+ internal double Memory() {
+ static double Pow2(int x) => Math.Pow(2, x);
+ var mem = Pow2(this.Header[2] + 2) + Pow2(this.Header[3]) + Pow2(this.Header[4] + 2) + Pow2(this.Header[5]) + this.Header.Length;
+ var cp = 7;
+ for (var i = 0; i < this.Header[6]; ++i) {
+ var size = Pow2(this.Header[cp + 1]);
+ switch (this.Header[cp]) {
+ case Cm: mem += 4 * size; break;
+ case Icm: mem += 64 * size + 1024; break;
+ case Match: mem += 4 * size + Pow2(this.Header[cp + 2]); break;
+ case Mix2: mem += 2 * size; break;
+ case Mix: mem += 4 * size * this.Header[cp + 3]; break;
+ case Isse: mem += 64 * size + 2048; break;
+ case Sse: mem += 128 * size; break;
+ }
+ cp += ComponentSize[this.Header[cp]];
+ }
+ return mem;
+ }
+
+ /// Writes the pending OUT bytes to and .
+ internal void Flush() {
+ var pending = this._outbuf.AsSpan(0, this._bufptr);
+ this.Output?.Write(pending);
+ this.Sha1?.Write(pending);
+ this._bufptr = 0;
+ }
+
+ /// Runs the program once with in A.
+ internal void Run(uint input) {
+ this._pc = this.Hbegin;
+ this._a = input;
+ while (this.Execute()) { }
+ }
+
+ /// Allocates a zero-filled array of 2^ elements of .
+ internal static T[] AllocatePow2(int bits, int extraShift = 0) {
+ var total = bits + extraShift;
+ if (total > 32)
+ ZpaqError.Throw("array too big");
+ var length = 1L << total;
+ if (length > Array.MaxLength)
+ throw new NotSupportedException($"ZPAQ: a model array of 2^{total} elements exceeds what this runtime can allocate.");
+ return new T[length];
+ }
+
+ private void Init(int hbits, int mbits) {
+ if (hbits > 32)
+ ZpaqError.Throw("H too big");
+ if (mbits > 32)
+ ZpaqError.Throw("M too big");
+ this._h = AllocatePow2(hbits);
+ this._m = AllocatePow2(mbits);
+ this._hMask = this._h.Length - 1;
+ this._mMask = this._m.Length - 1;
+ this._r = new uint[256];
+ this._a = this._b = this._c = this._d = 0;
+ this._pc = 0;
+ this._f = false;
+ }
+
+ /// Buffers output byte ; -1 flushes the buffer.
+ internal void OutC(int ch) {
+ if (ch < 0) {
+ this.Flush();
+ return;
+ }
+ this._outbuf[this._bufptr] = (byte)ch;
+ if (++this._bufptr == this._outbuf.Length)
+ this.Flush();
+ }
+
+ [System.Diagnostics.CodeAnalysis.DoesNotReturn]
+ private static void Error() => ZpaqError.Throw("ZPAQL execution error");
+
+ private void Div(uint x) => this._a = x != 0 ? this._a / x : 0;
+
+ private void Mod(uint x) => this._a = x != 0 ? this._a % x : 0;
+
+ private static void Swap(ref uint a, ref uint x) {
+ a ^= x;
+ x ^= a;
+ a ^= x;
+ }
+
+ private static void Swap(ref uint a, ref byte x) {
+ a ^= x;
+ x = (byte)(x ^ a);
+ a ^= x;
+ }
+
+ /// Interprets one instruction; returns after HALT.
+ private bool Execute() {
+ var header = this.Header;
+ var m = this._m;
+ var h = this._h;
+ var mm = this._mMask;
+ var hm = this._hMask;
+ ref var mb = ref m[(int)(this._b & (uint)mm)];
+ ref var mc = ref m[(int)(this._c & (uint)mm)];
+ ref var hd = ref h[(int)(this._d & (uint)hm)];
+ switch (header[this._pc++]) {
+ case 0: Error(); break;
+ case 1: ++this._a; break;
+ case 2: --this._a; break;
+ case 3: this._a = ~this._a; break;
+ case 4: this._a = 0; break;
+ case 7: this._a = this._r[header[this._pc++]]; break;
+ case 8: Swap(ref this._a, ref this._b); break;
+ case 9: ++this._b; break;
+ case 10: --this._b; break;
+ case 11: this._b = ~this._b; break;
+ case 12: this._b = 0; break;
+ case 15: this._b = this._r[header[this._pc++]]; break;
+ case 16: Swap(ref this._a, ref this._c); break;
+ case 17: ++this._c; break;
+ case 18: --this._c; break;
+ case 19: this._c = ~this._c; break;
+ case 20: this._c = 0; break;
+ case 23: this._c = this._r[header[this._pc++]]; break;
+ case 24: Swap(ref this._a, ref this._d); break;
+ case 25: ++this._d; break;
+ case 26: --this._d; break;
+ case 27: this._d = ~this._d; break;
+ case 28: this._d = 0; break;
+ case 31: this._d = this._r[header[this._pc++]]; break;
+ case 32: Swap(ref this._a, ref mb); break;
+ case 33: ++mb; break;
+ case 34: --mb; break;
+ case 35: mb = (byte)~mb; break;
+ case 36: mb = 0; break;
+ case 39: if (this._f) this._pc += ((header[this._pc] + 128) & 255) - 127; else ++this._pc; break;
+ case 40: Swap(ref this._a, ref mc); break;
+ case 41: ++mc; break;
+ case 42: --mc; break;
+ case 43: mc = (byte)~mc; break;
+ case 44: mc = 0; break;
+ case 47: if (!this._f) this._pc += ((header[this._pc] + 128) & 255) - 127; else ++this._pc; break;
+ case 48: Swap(ref this._a, ref hd); break;
+ case 49: ++hd; break;
+ case 50: --hd; break;
+ case 51: hd = ~hd; break;
+ case 52: hd = 0; break;
+ case 55: this._r[header[this._pc++]] = this._a; break;
+ case 56: return false;
+ case 57: this.OutC((int)(this._a & 255)); break;
+ case 59: this._a = (this._a + mb + 512) * 773; break;
+ case 60: hd = (hd + this._a + 512) * 773; break;
+ case 63: this._pc += ((header[this._pc] + 128) & 255) - 127; break;
+ case 64: break;
+ case 65: this._a = this._b; break;
+ case 66: this._a = this._c; break;
+ case 67: this._a = this._d; break;
+ case 68: this._a = mb; break;
+ case 69: this._a = mc; break;
+ case 70: this._a = hd; break;
+ case 71: this._a = header[this._pc++]; break;
+ case 72: this._b = this._a; break;
+ case 73: break;
+ case 74: this._b = this._c; break;
+ case 75: this._b = this._d; break;
+ case 76: this._b = mb; break;
+ case 77: this._b = mc; break;
+ case 78: this._b = hd; break;
+ case 79: this._b = header[this._pc++]; break;
+ case 80: this._c = this._a; break;
+ case 81: this._c = this._b; break;
+ case 82: break;
+ case 83: this._c = this._d; break;
+ case 84: this._c = mb; break;
+ case 85: this._c = mc; break;
+ case 86: this._c = hd; break;
+ case 87: this._c = header[this._pc++]; break;
+ case 88: this._d = this._a; break;
+ case 89: this._d = this._b; break;
+ case 90: this._d = this._c; break;
+ case 91: break;
+ case 92: this._d = mb; break;
+ case 93: this._d = mc; break;
+ case 94: this._d = hd; break;
+ case 95: this._d = header[this._pc++]; break;
+ case 96: mb = (byte)this._a; break;
+ case 97: mb = (byte)this._b; break;
+ case 98: mb = (byte)this._c; break;
+ case 99: mb = (byte)this._d; break;
+ case 100: break;
+ case 101: mb = mc; break;
+ case 102: mb = (byte)hd; break;
+ case 103: mb = header[this._pc++]; break;
+ case 104: mc = (byte)this._a; break;
+ case 105: mc = (byte)this._b; break;
+ case 106: mc = (byte)this._c; break;
+ case 107: mc = (byte)this._d; break;
+ case 108: mc = mb; break;
+ case 109: break;
+ case 110: mc = (byte)hd; break;
+ case 111: mc = header[this._pc++]; break;
+ case 112: hd = this._a; break;
+ case 113: hd = this._b; break;
+ case 114: hd = this._c; break;
+ case 115: hd = this._d; break;
+ case 116: hd = mb; break;
+ case 117: hd = mc; break;
+ case 118: break;
+ case 119: hd = header[this._pc++]; break;
+ case 128: this._a += this._a; break;
+ case 129: this._a += this._b; break;
+ case 130: this._a += this._c; break;
+ case 131: this._a += this._d; break;
+ case 132: this._a += mb; break;
+ case 133: this._a += mc; break;
+ case 134: this._a += hd; break;
+ case 135: this._a += header[this._pc++]; break;
+ case 136: this._a = 0; break;
+ case 137: this._a -= this._b; break;
+ case 138: this._a -= this._c; break;
+ case 139: this._a -= this._d; break;
+ case 140: this._a -= mb; break;
+ case 141: this._a -= mc; break;
+ case 142: this._a -= hd; break;
+ case 143: this._a -= header[this._pc++]; break;
+ case 144: this._a *= this._a; break;
+ case 145: this._a *= this._b; break;
+ case 146: this._a *= this._c; break;
+ case 147: this._a *= this._d; break;
+ case 148: this._a *= mb; break;
+ case 149: this._a *= mc; break;
+ case 150: this._a *= hd; break;
+ case 151: this._a *= header[this._pc++]; break;
+ case 152: this.Div(this._a); break;
+ case 153: this.Div(this._b); break;
+ case 154: this.Div(this._c); break;
+ case 155: this.Div(this._d); break;
+ case 156: this.Div(mb); break;
+ case 157: this.Div(mc); break;
+ case 158: this.Div(hd); break;
+ case 159: this.Div(header[this._pc++]); break;
+ case 160: this.Mod(this._a); break;
+ case 161: this.Mod(this._b); break;
+ case 162: this.Mod(this._c); break;
+ case 163: this.Mod(this._d); break;
+ case 164: this.Mod(mb); break;
+ case 165: this.Mod(mc); break;
+ case 166: this.Mod(hd); break;
+ case 167: this.Mod(header[this._pc++]); break;
+ case 168: break;
+ case 169: this._a &= this._b; break;
+ case 170: this._a &= this._c; break;
+ case 171: this._a &= this._d; break;
+ case 172: this._a &= mb; break;
+ case 173: this._a &= mc; break;
+ case 174: this._a &= hd; break;
+ case 175: this._a &= header[this._pc++]; break;
+ case 176: this._a = 0; break;
+ case 177: this._a &= ~this._b; break;
+ case 178: this._a &= ~this._c; break;
+ case 179: this._a &= ~this._d; break;
+ case 180: this._a &= ~(uint)mb; break;
+ case 181: this._a &= ~(uint)mc; break;
+ case 182: this._a &= ~hd; break;
+ case 183: this._a &= ~(uint)header[this._pc++]; break;
+ case 184: break;
+ case 185: this._a |= this._b; break;
+ case 186: this._a |= this._c; break;
+ case 187: this._a |= this._d; break;
+ case 188: this._a |= mb; break;
+ case 189: this._a |= mc; break;
+ case 190: this._a |= hd; break;
+ case 191: this._a |= header[this._pc++]; break;
+ case 192: this._a = 0; break;
+ case 193: this._a ^= this._b; break;
+ case 194: this._a ^= this._c; break;
+ case 195: this._a ^= this._d; break;
+ case 196: this._a ^= mb; break;
+ case 197: this._a ^= mc; break;
+ case 198: this._a ^= hd; break;
+ case 199: this._a ^= header[this._pc++]; break;
+ case 200: this._a <<= (int)(this._a & 31); break;
+ case 201: this._a <<= (int)(this._b & 31); break;
+ case 202: this._a <<= (int)(this._c & 31); break;
+ case 203: this._a <<= (int)(this._d & 31); break;
+ case 204: this._a <<= mb & 31; break;
+ case 205: this._a <<= mc & 31; break;
+ case 206: this._a <<= (int)(hd & 31); break;
+ case 207: this._a <<= header[this._pc++] & 31; break;
+ case 208: this._a >>= (int)(this._a & 31); break;
+ case 209: this._a >>= (int)(this._b & 31); break;
+ case 210: this._a >>= (int)(this._c & 31); break;
+ case 211: this._a >>= (int)(this._d & 31); break;
+ case 212: this._a >>= mb & 31; break;
+ case 213: this._a >>= mc & 31; break;
+ case 214: this._a >>= (int)(hd & 31); break;
+ case 215: this._a >>= header[this._pc++] & 31; break;
+ case 216: this._f = true; break;
+ case 217: this._f = this._a == this._b; break;
+ case 218: this._f = this._a == this._c; break;
+ case 219: this._f = this._a == this._d; break;
+ case 220: this._f = this._a == mb; break;
+ case 221: this._f = this._a == mc; break;
+ case 222: this._f = this._a == hd; break;
+ case 223: this._f = this._a == header[this._pc++]; break;
+ case 224: this._f = false; break;
+ case 225: this._f = this._a < this._b; break;
+ case 226: this._f = this._a < this._c; break;
+ case 227: this._f = this._a < this._d; break;
+ case 228: this._f = this._a < mb; break;
+ case 229: this._f = this._a < mc; break;
+ case 230: this._f = this._a < hd; break;
+ case 231: this._f = this._a < header[this._pc++]; break;
+ case 232: this._f = false; break;
+ case 233: this._f = this._a > this._b; break;
+ case 234: this._f = this._a > this._c; break;
+ case 235: this._f = this._a > this._d; break;
+ case 236: this._f = this._a > mb; break;
+ case 237: this._f = this._a > mc; break;
+ case 238: this._f = this._a > hd; break;
+ case 239: this._f = this._a > header[this._pc++]; break;
+ case 255:
+ if ((this._pc = this.Hbegin + header[this._pc] + 256 * header[this._pc + 1]) >= this.Hend)
+ Error();
+ break;
+ default: Error(); break;
+ }
+ return true;
+ }
+}
diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqlVm.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqlVm.cs
deleted file mode 100644
index 6f731ba25..000000000
--- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/ZpaqlVm.cs
+++ /dev/null
@@ -1,960 +0,0 @@
-using System.Runtime.CompilerServices;
-
-namespace FileFormat.Zpaq;
-
-///
-/// Component type identifiers for the ZPAQL context-mixing prediction model.
-///
-public enum ComponentType : byte {
- /// Constant prediction (always predicts 50/50).
- Const = 0,
-
- /// Context model — 2^n counters indexed by hashed context.
- Cm = 1,
-
- /// Indirect context model — byte history maps to a counter.
- Icm = 2,
-
- /// Match model — predicts by finding a previous matching context.
- Match = 3,
-
- /// Average of two component predictions.
- Avg = 4,
-
- /// Two-input adaptive mixer.
- Mix2 = 5,
-
- /// N-input adaptive mixer.
- Mix = 6,
-
- /// Indirect secondary symbol estimation.
- Isse = 7,
-
- /// Direct secondary symbol estimation.
- Sse = 8,
-}
-
-///
-/// Represents a single prediction component in the ZPAQL context-mixing model.
-///
-public sealed class Component {
- /// The component type.
- public ComponentType Type { get; set; }
-
- /// First parameter (meaning depends on type, e.g. log2 of table size).
- public int Param1 { get; set; }
-
- /// Second parameter (meaning depends on type).
- public int Param2 { get; set; }
-
- /// Third parameter (meaning depends on type).
- public int Param3 { get; set; }
-
- /// Component memory (CM counters, ICM byte histories, etc.).
- public int[] Memory { get; set; } = [];
-
- /// Current hashed context value.
- public int Context { get; set; }
-
- /// Current prediction (0..65535, probability of next bit being 1).
- public int Prediction { get; set; } = 32768; // 50%
-
- /// Auxiliary state (match length, mixer weights, etc.).
- public int Aux { get; set; }
-}
-
-///
-/// ZPAQL virtual machine for ZPAQ compression and decompression.
-///
-///
-///
-/// The ZPAQL VM is a bytecode interpreter with four 32-bit registers (A, B, C, D),
-/// a 1-bit flag register (F), a 256-element register file (R[0]..R[255]),
-/// a 32-bit H[] array, and an 8-bit M[] array. It executes two programs:
-///
-///
-/// -
-/// HCOMP — runs after each decoded byte to update hash contexts in H[].
-///
-/// -
-/// PCOMP — optional post-processor that transforms decoded output.
-///
-///
-///
-/// The VM also manages an array of context-mixing components (CM, ICM, MATCH, etc.)
-/// and combines their predictions into a single probability for the next bit. Coding
-/// that bit against the probability is the job of and
-/// , which are deliberately separate: the VM decides
-/// what the odds are, the coder turns them into bytes.
-///
-///
-public sealed class ZpaqlVm {
- // ── Registers ───────────────────────────────────────────────────────────────
-
- private uint _a;
- private uint _b;
- private uint _c;
- private uint _d;
- private bool _f;
- private readonly uint[] _r = new uint[256];
- private int _pc;
-
- // ── Memory arrays ──────────────────────────────────────────────────────────
-
- private uint[] _h; // H array (32-bit), indexed by B & (length-1)
- private byte[] _m; // M array (8-bit), indexed by B/C/D & (length-1)
- private int _hMask; // H.Length - 1 for wrapping
- private int _mMask; // M.Length - 1 for wrapping
-
- // ── Programs ───────────────────────────────────────────────────────────────
-
- private readonly byte[] _hcomp; // HCOMP bytecode
- private readonly byte[] _pcomp; // PCOMP bytecode (optional, may be empty)
-
- // ── Context-mixing components ──────────────────────────────────────────────
-
- private readonly Component[] _components;
-
- // ── Squash / stretch tables for probability mapping ────────────────────────
-
- private static readonly int[] SquashTable = BuildSquashTable();
- private static readonly int[] StretchTable = BuildStretchTable();
-
- // ── Construction ───────────────────────────────────────────────────────────
-
- ///
- /// Creates a ZPAQL VM with the specified program and component configuration.
- ///
- /// HCOMP bytecode (context hashing program).
- /// PCOMP bytecode (post-processing program, may be empty).
- /// Log2 of H[] array size (0..31). H has 2^hh entries.
- /// Log2 of M[] array size (0..31). M has 2^hm entries.
- /// Context-mixing component definitions. May be empty for store-only archives.
- /// Thrown if is null.
- public ZpaqlVm(byte[] hcomp, byte[] pcomp, int hh, int hm, Component[]? components = null) {
- ArgumentNullException.ThrowIfNull(hcomp);
- _hcomp = hcomp;
- _pcomp = pcomp ?? [];
- _components = components ?? [];
-
- hh = Math.Clamp(hh, 0, 24);
- hm = Math.Clamp(hm, 0, 24);
- var hSize = 1 << hh;
- var mSize = 1 << hm;
- _h = new uint[hSize];
- _m = new byte[mSize];
- _hMask = hSize - 1;
- _mMask = mSize - 1;
-
- InitComponents();
- }
-
- ///
- /// Parses a ZPAQL program header from a binary span and constructs the VM.
- ///
- ///
- /// Binary ZPAQL program header. Layout:
- /// [0]: hh (log2 H size), [1]: hm (log2 M size),
- /// [2]: number of components (n),
- /// followed by n component descriptors (type + params),
- /// then HCOMP bytecode (2-byte LE length + code),
- /// then optional PCOMP bytecode (2-byte LE length + code, 0 if absent).
- ///
- /// Thrown if the program header is malformed.
- public ZpaqlVm(ReadOnlySpan program) {
- if (program.Length < 3)
- ThrowMalformed("Program header too short.");
-
- var hh = Math.Clamp((int)program[0], 0, 24);
- var hm = Math.Clamp((int)program[1], 0, 24);
- int nComp = program[2];
- var pos = 3;
-
- // Parse component descriptors.
- var comps = new List();
- for (var i = 0; i < nComp; i++) {
- if (pos >= program.Length)
- ThrowMalformed("Unexpected end of component descriptors.");
-
- var comp = new Component { Type = (ComponentType)program[pos++] };
-
- // Each component type has a fixed number of parameter bytes.
- var paramCount = GetComponentParamCount(comp.Type);
- if (pos + paramCount > program.Length)
- ThrowMalformed("Unexpected end of component parameters.");
-
- if (paramCount >= 1) comp.Param1 = program[pos++];
- if (paramCount >= 2) comp.Param2 = program[pos++];
- if (paramCount >= 3) comp.Param3 = program[pos++];
-
- comps.Add(comp);
- }
-
- _components = comps.ToArray();
-
- // Parse HCOMP bytecode.
- if (pos + 2 > program.Length)
- ThrowMalformed("Missing HCOMP length.");
- var hcompLen = program[pos] | (program[pos + 1] << 8);
- pos += 2;
- if (pos + hcompLen > program.Length)
- ThrowMalformed("HCOMP bytecode truncated.");
- _hcomp = program.Slice(pos, hcompLen).ToArray();
- pos += hcompLen;
-
- // Parse optional PCOMP bytecode.
- if (pos + 2 <= program.Length) {
- var pcompLen = program[pos] | (program[pos + 1] << 8);
- pos += 2;
- if (pcompLen > 0 && pos + pcompLen <= program.Length) {
- _pcomp = program.Slice(pos, pcompLen).ToArray();
- } else {
- _pcomp = [];
- }
- } else {
- _pcomp = [];
- }
-
- var hSize = 1 << hh;
- var mSize = 1 << hm;
- _h = new uint[hSize];
- _m = new byte[mSize];
- _hMask = hSize - 1;
- _mMask = mSize - 1;
-
- InitComponents();
- }
-
- // ── Public API ─────────────────────────────────────────────────────────────
-
- ///
- /// Gets the H[] context array. H values are set by the HCOMP program and
- /// used as context indices for the prediction components.
- ///
- public uint[] H => _h;
-
- ///
- /// Gets the M[] byte array used by the HCOMP/PCOMP programs.
- ///
- public byte[] M => _m;
-
- ///
- /// Gets the context-mixing components.
- ///
- public Component[] Components => _components;
-
- ///
- /// Gets or sets register A (32-bit accumulator).
- ///
- public uint A { get => _a; set => _a = value; }
-
- ///
- /// Gets or sets register B (32-bit, also indexes M[]).
- ///
- public uint B { get => _b; set => _b = value; }
-
- ///
- /// Gets or sets register C (32-bit, also indexes M[]).
- ///
- public uint C { get => _c; set => _c = value; }
-
- ///
- /// Gets or sets register D (32-bit, also indexes M[]).
- ///
- public uint D { get => _d; set => _d = value; }
-
- ///
- /// Gets or sets the flag register (1-bit).
- ///
- public bool F { get => _f; set => _f = value; }
-
- ///
- /// Executes the HCOMP program with the given input byte.
- /// This should be called after each decoded byte to update the
- /// context hashes in H[].
- ///
- /// The decoded byte value (0..255), or -1 for EOF.
- public void RunHcomp(int input) {
- Execute(_hcomp, input);
- }
-
- ///
- /// Executes the PCOMP (post-processing) program with the given input byte.
- ///
- /// The decoded byte value (0..255), or -1 for EOF.
- public void RunPcomp(int input) {
- if (_pcomp.Length > 0)
- Execute(_pcomp, input);
- }
-
- ///
- /// Computes a combined prediction from all components.
- ///
- ///
- /// The components are averaged with equal weight, in ascending component order,
- /// and the sum is divided with truncation towards zero. Both the order and the
- /// rounding are part of the coded stream, so neither may be left to a container's
- /// enumeration order. The result is clamped into the range the range coder accepts,
- /// so that both of its subranges stay non-empty; see .
- ///
- /// Probability that the next bit is 1, in the range 1..65535.
- public int Predict() {
- if (_components.Length == 0)
- return 32768; // 50% if no components
-
- // Simple averaging of all component predictions.
- // A full implementation would use the mixer components (MIX, MIX2)
- // from the COMP section for weighted mixing.
- long sum = 0;
-
- for (var i = 0; i < _components.Length; i++) {
- var comp = _components[i];
- UpdateComponentPrediction(comp, i);
- sum += comp.Prediction;
- }
-
- return (int)Math.Clamp(
- sum / _components.Length,
- ZpaqRangeCoder.MinimumProbability,
- ZpaqRangeCoder.MaximumProbability);
- }
-
- ///
- /// Updates all components after a decoded bit.
- ///
- /// The decoded bit (0 or 1).
- public void Update(int bit) {
- for (var i = 0; i < _components.Length; i++)
- UpdateComponent(_components[i], bit);
- }
-
- ///
- /// Resets the VM state (registers, arrays, components) without changing the
- /// program or component configuration.
- ///
- public void Reset() {
- _a = _b = _c = _d = 0;
- _f = false;
- Array.Clear(_r);
- Array.Clear(_h);
- Array.Clear(_m);
-
- InitComponents();
- }
-
- // ── Bytecode execution ─────────────────────────────────────────────────────
-
- private void Execute(byte[] code, int inputByte) {
- if (code.Length == 0)
- return;
-
- // Place input in A register as per ZPAQL spec.
- _a = (uint)(inputByte & 0xFF);
- _pc = 0;
- var limit = 1 << 24; // execution limit to prevent infinite loops
-
- while (_pc < code.Length && --limit > 0) {
- var op = code[_pc++];
-
- switch (op) {
- // 0: ERROR / HALT
- case 0:
- return;
-
- // 2..3: A++, A--
- case 2: _a++; break;
- case 3: _a--; break;
-
- // 4: A = !A (logical NOT)
- case 4: _a = _a == 0 ? 1U : 0U; break;
-
- // 5: A = 0
- case 5: _a = 0; break;
-
- // 6..8: B=A, C=A, D=A
- case 6: _b = _a; break;
- case 7: _c = _a; break;
- case 8: _d = _a; break;
-
- // 9..11: *B=A, *C=A, *D=A (write to M[])
- case 9: _m[(int)(_b & (uint)_mMask)] = (byte)_a; break;
- case 10: _m[(int)(_c & (uint)_mMask)] = (byte)_a; break;
- case 11: _m[(int)(_d & (uint)_mMask)] = (byte)_a; break;
-
- // 12..14: A=B, A=C, A=D
- case 12: _a = _b; break;
- case 13: _a = _c; break;
- case 14: _a = _d; break;
-
- // 15..17: A=*B, A=*C, A=*D (read from M[])
- case 15: _a = _m[(int)(_b & (uint)_mMask)]; break;
- case 16: _a = _m[(int)(_c & (uint)_mMask)]; break;
- case 17: _a = _m[(int)(_d & (uint)_mMask)]; break;
-
- // 18: A = B + A, F = (result == 0)
- case 18: _a = _b + _a; _f = _a == 0; break;
-
- // 19: A = B - A, F = borrow (B < A before subtraction)
- case 19: _f = _b < _a; _a = _b - _a; break;
-
- // 20: A = B * A, F = (result == 0)
- case 20: _a = _b * _a; _f = _a == 0; break;
-
- // 21: A = B / A, F = (remainder == 0). Div by 0 => A = 0.
- case 21:
- if (_a == 0) { _a = 0; _f = true; }
- else { var rem = _b % _a; _a = _b / _a; _f = rem == 0; }
- break;
-
- // 22: A = B % A. Div by 0 => A = 0.
- case 22:
- _a = _a == 0 ? 0 : _b % _a;
- break;
-
- // 23..25: A = B & A, B | A, B ^ A
- case 23: _a = _b & _a; break;
- case 24: _a = _b | _a; break;
- case 25: _a = _b ^ _a; break;
-
- // 26: A = B << (A mod 32)
- case 26: _a = _b << (int)(_a & 31); break;
-
- // 27: A = B >> (A mod 32) (logical)
- case 27: _a = _b >> (int)(_a & 31); break;
-
- // 28: A <<= 8
- case 28: _a <<= 8; break;
-
- // 29: A >>= 8 (logical)
- case 29: _a >>= 8; break;
-
- // 30: F = (A == B)
- case 30: _f = _a == _b; break;
-
- // 31: F = (A < B) unsigned
- case 31: _f = _a < _b; break;
-
- // 32: F = (A > B) unsigned
- case 32: _f = _a > _b; break;
-
- // 33..38: reserved / unused
- case 33: case 34: case 35: case 36: case 37: case 38:
- break;
-
- // 39: HALT
- case 39: return;
-
- // 40: F? (no-op — just tests F, used as prefix)
- case 40: break;
-
- // 41: JT — jump if F is true (1-byte signed offset follows)
- case 41: {
- var offset = ReadSignedByte(code, ref _pc);
- if (_f) _pc += offset;
- break;
- }
-
- // 42: JF — jump if F is false (1-byte signed offset follows)
- case 42: {
- var offset = ReadSignedByte(code, ref _pc);
- if (!_f) _pc += offset;
- break;
- }
-
- // 43: JMP — unconditional jump (1-byte signed offset)
- case 43: {
- var offset = ReadSignedByte(code, ref _pc);
- _pc += offset;
- break;
- }
-
- // 44..46: reserved jump variants
- case 44: case 45: case 46:
- if (_pc < code.Length) _pc++; // skip 1-byte operand
- break;
-
- // 47: LJ — long jump (2-byte unsigned absolute target)
- case 47: {
- var target = ReadUInt16(code, ref _pc);
- _pc = target;
- break;
- }
-
- // 48: R[imm] = A (immediate register index follows)
- case 48: {
- var idx = (int)ReadByte(code, ref _pc);
- _r[idx] = _a;
- break;
- }
-
- // 49: A = R[imm]
- case 49: {
- var idx = (int)ReadByte(code, ref _pc);
- _a = _r[idx];
- break;
- }
-
- // 50..55: reserved register ops (with 1-byte operand)
- case 50: case 51: case 52: case 53: case 54: case 55:
- if (_pc < code.Length) _pc++; // skip operand
- break;
-
- // 56: A += imm
- case 56: _a += ReadByte(code, ref _pc); break;
-
- // 57: A -= imm
- case 57: _a -= ReadByte(code, ref _pc); break;
-
- // 58: A *= imm
- case 58: _a *= ReadByte(code, ref _pc); break;
-
- // 59: A /= imm (0 => A=0)
- case 59: {
- var imm = ReadByte(code, ref _pc);
- _a = imm == 0 ? 0 : _a / imm;
- break;
- }
-
- // 60: A %= imm (0 => A=0)
- case 60: {
- var imm = ReadByte(code, ref _pc);
- _a = imm == 0 ? 0 : _a % imm;
- break;
- }
-
- // 61: A &= imm
- case 61: _a &= ReadByte(code, ref _pc); break;
-
- // 62: A |= imm
- case 62: _a |= ReadByte(code, ref _pc); break;
-
- // 63: A ^= imm
- case 63: _a ^= ReadByte(code, ref _pc); break;
-
- // 64..127: 2-byte instructions
- // Most are A op= imm16 or extended register/memory ops.
- case >= 64 and <= 127: {
- Execute2Byte(op, code);
- break;
- }
-
- // 128..223: 3-byte instructions (conditional jumps with 2-byte offset, etc.)
- case >= 128 and <= 223: {
- Execute3Byte(op, code);
- break;
- }
-
- // 224: OUT — emit A as output byte (used in PCOMP)
- case 224:
- OutputByte?.Invoke((byte)_a);
- break;
-
- // 225: HASH — A = (A + *B + 512) * 773
- case 225: {
- uint mb = _m[(int)(_b & (uint)_mMask)];
- _a = (_a + mb + 512) * 773;
- break;
- }
-
- // 226: HASHD — H[D] = (H[D] + A + 512) * 773
- case 226: {
- var idx = (int)(_d & (uint)_hMask);
- _h[idx] = (_h[idx] + _a + 512) * 773;
- break;
- }
-
- // 227..254: reserved / unused
- case >= 227 and <= 254:
- break;
-
- // 255: HALT
- case 255: return;
- }
- }
- }
-
- ///
- /// Event invoked when the PCOMP program executes an OUT instruction.
- ///
- public Action? OutputByte { get; set; }
-
- // ── 2-byte instruction handler ─────────────────────────────────────────────
-
- private void Execute2Byte(byte op, byte[] code) {
- // Opcodes 64..127: the second byte is an immediate operand.
- // We handle a useful subset; unrecognized opcodes skip 1 byte.
-
- var imm = ReadByte(code, ref _pc);
-
- switch (op) {
- // 64: A += imm*256 (upper byte add)
- case 64: _a += imm << 8; break;
-
- // 65: A -= imm*256
- case 65: _a -= imm << 8; break;
-
- // 66: A *= imm*256
- case 66: _a *= imm << 8; break;
-
- // 67: A /= imm*256 (0 => A=0)
- case 67: { var v = imm << 8; _a = v == 0 ? 0 : _a / v; break; }
-
- // 68: A %= imm*256 (0 => A=0)
- case 68: { var v = imm << 8; _a = v == 0 ? 0 : _a % v; break; }
-
- // 69: A &= imm*256
- case 69: _a &= imm << 8; break;
-
- // 70: A |= imm*256
- case 70: _a |= imm << 8; break;
-
- // 71: A ^= imm*256
- case 71: _a ^= imm << 8; break;
-
- // 72: A = imm (set A to 1-byte immediate)
- case 72: _a = imm; break;
-
- // 73: B = imm
- case 73: _b = imm; break;
-
- // 74: C = imm
- case 74: _c = imm; break;
-
- // 75: D = imm
- case 75: _d = imm; break;
-
- // 76: A += imm*65536
- case 76: _a += imm << 16; break;
-
- // 77: A -= imm*65536
- case 77: _a -= imm << 16; break;
-
- // 78: A += imm*16777216
- case 78: _a += imm << 24; break;
-
- // 79: A -= imm*16777216
- case 79: _a -= imm << 24; break;
-
- // 80: H[B] = A (indirect H store, imm ignored)
- case 80: _h[(int)(_b & (uint)_hMask)] = _a; break;
-
- // 81: A = H[B] (indirect H load, imm ignored)
- case 81: _a = _h[(int)(_b & (uint)_hMask)]; break;
-
- // 82: H[imm] = A
- case 82: _h[(int)(imm & (uint)_hMask)] = _a; break;
-
- // 83: A = H[imm]
- case 83: _a = _h[(int)(imm & (uint)_hMask)]; break;
-
- // Everything else: operand already consumed, no-op.
- default: break;
- }
- }
-
- // ── 3-byte instruction handler ─────────────────────────────────────────────
-
- private void Execute3Byte(byte op, byte[] code) {
- // Opcodes 128..223: 2-byte operand follows (little-endian).
- var imm16 = (int)ReadUInt16(code, ref _pc);
-
- switch (op) {
- // 128: A = imm16
- case 128: _a = (uint)imm16; break;
-
- // 129: B = imm16
- case 129: _b = (uint)imm16; break;
-
- // 130: C = imm16
- case 130: _c = (uint)imm16; break;
-
- // 131: D = imm16
- case 131: _d = (uint)imm16; break;
-
- // 132: JT with 2-byte offset
- case 132: if (_f) _pc = imm16; break;
-
- // 133: JF with 2-byte offset
- case 133: if (!_f) _pc = imm16; break;
-
- // 134: JMP with 2-byte absolute target
- case 134: _pc = imm16; break;
-
- // 135: A += imm16
- case 135: _a += (uint)imm16; break;
-
- // 136: A -= imm16
- case 136: _a -= (uint)imm16; break;
-
- // 137: A *= imm16
- case 137: _a *= (uint)imm16; break;
-
- // 138: A /= imm16
- case 138: _a = imm16 == 0 ? 0 : _a / (uint)imm16; break;
-
- // 139: A %= imm16
- case 139: _a = imm16 == 0 ? 0 : _a % (uint)imm16; break;
-
- // 140: A &= imm16
- case 140: _a &= (uint)imm16; break;
-
- // 141: A |= imm16
- case 141: _a |= (uint)imm16; break;
-
- // 142: A ^= imm16
- case 142: _a ^= (uint)imm16; break;
-
- // Everything else: operand already consumed, no-op.
- default: break;
- }
- }
-
- // ── Operand reading helpers ────────────────────────────────────────────────
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static uint ReadByte(byte[] code, ref int pc) =>
- pc < code.Length ? code[pc++] : 0U;
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static int ReadSignedByte(byte[] code, ref int pc) =>
- pc < code.Length ? (sbyte)code[pc++] : 0;
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static int ReadUInt16(byte[] code, ref int pc) {
- if (pc + 1 >= code.Length) {
- pc = code.Length;
- return 0;
- }
- var val = code[pc] | (code[pc + 1] << 8);
- pc += 2;
- return val;
- }
-
- // ── Component initialization ───────────────────────────────────────────────
-
- private void InitComponents() {
- for (var i = 0; i < _components.Length; i++) {
- var comp = _components[i];
- comp.Prediction = 32768; // 50%
- comp.Context = 0;
- comp.Aux = 0;
-
- switch (comp.Type) {
- case ComponentType.Cm: {
- // 2^Param1 counters, each starts at 32768 (50%).
- var size = 1 << Math.Clamp(comp.Param1, 0, 24);
- comp.Memory = new int[size];
- for (var j = 0; j < size; j++)
- comp.Memory[j] = 32768;
- break;
- }
-
- case ComponentType.Icm: {
- // Indirect context model: 256 byte-history entries + 2^(Param1+10) counters.
- var histSize = 256;
- var ctrSize = 1 << Math.Clamp(comp.Param1 + 10, 0, 24);
- comp.Memory = new int[histSize + ctrSize];
- // Initialize counters to 50%.
- for (var j = histSize; j < comp.Memory.Length; j++)
- comp.Memory[j] = 32768;
- break;
- }
-
- case ComponentType.Match: {
- // Match model: buffer of 2^Param1 bytes + 2^Param2 hash entries.
- var bufSize = 1 << Math.Clamp(comp.Param1, 0, 24);
- var hashSize = 1 << Math.Clamp(comp.Param2, 0, 24);
- comp.Memory = new int[bufSize + hashSize];
- break;
- }
-
- case ComponentType.Sse: {
- // SSE: 2^Param1 * 64 entries.
- var size = (1 << Math.Clamp(comp.Param1, 0, 20)) * 64;
- comp.Memory = new int[size];
- // Initialize to identity mapping.
- for (var j = 0; j < size; j++)
- comp.Memory[j] = Squash((j % 64 - 32) * 64);
- break;
- }
-
- default:
- comp.Memory ??= [];
- break;
- }
- }
- }
-
- // ── Component prediction & update ──────────────────────────────────────────
-
- private void UpdateComponentPrediction(Component comp, int index) {
- // Set context from H[index].
- comp.Context = (int)(_h[index & _hMask] & 0x7FFFFFFF);
-
- switch (comp.Type) {
- case ComponentType.Const:
- comp.Prediction = 32768;
- break;
-
- case ComponentType.Cm:
- if (comp.Memory.Length > 0) {
- var ctx = comp.Context % comp.Memory.Length;
- comp.Prediction = Math.Clamp(comp.Memory[ctx], 0, 65535);
- }
- break;
-
- case ComponentType.Icm:
- if (comp.Memory.Length > 256) {
- var histIdx = comp.Context & 255;
- var ctrIdx = 256 + (comp.Memory[histIdx] & (comp.Memory.Length - 257));
- if (ctrIdx < comp.Memory.Length)
- comp.Prediction = Math.Clamp(comp.Memory[ctrIdx], 0, 65535);
- }
- break;
-
- case ComponentType.Match:
- // Match model: if currently matching, predict from match bit.
- if (comp.Aux > 0)
- comp.Prediction = comp.Aux > 128 ? 65535 - 256 : 256;
- else
- comp.Prediction = 32768;
- break;
-
- case ComponentType.Avg:
- if (_components.Length > Math.Max(comp.Param1, comp.Param2)) {
- var p1 = _components[comp.Param1].Prediction;
- var p2 = _components[comp.Param2].Prediction;
- comp.Prediction = (p1 + p2 + 1) >> 1;
- }
- break;
-
- case ComponentType.Sse:
- if (comp.Memory.Length > 0) {
- var ctx = (comp.Context * 64) % comp.Memory.Length;
- comp.Prediction = Math.Clamp(comp.Memory[ctx], 0, 65535);
- }
- break;
-
- default:
- comp.Prediction = 32768;
- break;
- }
- }
-
- private static void UpdateComponent(Component comp, int bit) {
- switch (comp.Type) {
- case ComponentType.Cm:
- if (comp.Memory.Length > 0) {
- var ctx = comp.Context % comp.Memory.Length;
- var p = comp.Memory[ctx];
- // Adapt towards the observed bit by a fixed fraction of the remaining
- // error. The shift is arithmetic, so the step is the error divided by
- // 2^rate rounded towards negative infinity — that asymmetry is what
- // lets a counter reach 0 exactly while it only approaches 65535, and
- // it is part of the coded stream, so it must be reproduced as a floor
- // division rather than a truncating one.
- var target = bit != 0 ? 65535 : 0;
- var rate = Math.Max(comp.Param2, 2); // learning rate
- comp.Memory[ctx] = p + ((target - p) >> rate);
- }
- break;
-
- case ComponentType.Icm:
- if (comp.Memory.Length > 256) {
- var histIdx = comp.Context & 255;
- var ctrIdx = 256 + (comp.Memory[histIdx] & (comp.Memory.Length - 257));
- if (ctrIdx < comp.Memory.Length) {
- var p = comp.Memory[ctrIdx];
- var target = bit != 0 ? 65535 : 0;
- comp.Memory[ctrIdx] = p + ((target - p) >> 5);
- }
- // Update byte history: shift in bit.
- comp.Memory[histIdx] = (comp.Memory[histIdx] << 1) | bit;
- }
- break;
-
- case ComponentType.Match:
- // Update match length tracking.
- if (comp.Aux > 0) {
- // If prediction was wrong, break match.
- var predicted1 = comp.Aux > 128;
- if ((bit != 0) != predicted1)
- comp.Aux = 0;
- }
- break;
-
- // CONST, AVG, SSE, MIX, MIX2, ISSE: simplified — no update needed for basic operation.
- default:
- break;
- }
- }
-
- // ── Probability tables ─────────────────────────────────────────────────────
-
- ///
- /// Squash function: maps a logistic value (-2048..2047) to a probability (0..65535).
- /// squash(x) = 65536 / (1 + exp(-x / 64)).
- ///
- /// Logistic value.
- /// Probability in range 1..65534.
- public static int Squash(int x) {
- x = Math.Clamp(x, -2047, 2047);
- return SquashTable[x + 2047];
- }
-
- ///
- /// Stretch function: inverse of squash, maps probability (1..65534) to logistic.
- ///
- /// Probability (1..65534).
- /// Logistic value approximately in range -2047..2047.
- public static int Stretch(int p) {
- p = Math.Clamp(p, 1, 65534);
- return StretchTable[p >> 6]; // 1024-entry table
- }
-
- private static int[] BuildSquashTable() {
- // squash(x) = 65536 / (1 + exp(-x/64))
- var table = new int[4095]; // index 0..4094 maps x = -2047..2047
- for (var i = 0; i < table.Length; i++) {
- var x = i - 2047;
- var p = 65536.0 / (1.0 + Math.Exp(-x / 64.0));
- table[i] = Math.Clamp((int)(p + 0.5), 1, 65534);
- }
- return table;
- }
-
- private static int[] BuildStretchTable() {
- // Inverse of squash, sampled at 1024 points.
- var table = new int[1024];
- for (var i = 0; i < 1024; i++) {
- var p = (i + 0.5) * 64.0; // map to 0..65535 range
- if (p <= 0) p = 1;
- if (p >= 65535) p = 65534;
- var x = -64.0 * Math.Log(65536.0 / p - 1.0);
- table[i] = Math.Clamp((int)Math.Round(x), -2047, 2047);
- }
- return table;
- }
-
- // ── Component parameter counts ─────────────────────────────────────────────
-
- private static int GetComponentParamCount(ComponentType type) =>
- type switch {
- ComponentType.Const => 0,
- ComponentType.Cm => 2, // size, rate
- ComponentType.Icm => 1, // size
- ComponentType.Match => 2, // bufSize, hashSize
- ComponentType.Avg => 2, // comp1, comp2
- ComponentType.Mix2 => 3, // context, comp1, comp2
- ComponentType.Mix => 3, // context, nInputs, rate
- ComponentType.Isse => 2, // context, size
- ComponentType.Sse => 2, // context, size
- _ => 0,
- };
-
- // ── Throw helpers ──────────────────────────────────────────────────────────
-
- [MethodImpl(MethodImplOptions.NoInlining)]
- private static void ThrowMalformed(string message) =>
- throw new InvalidDataException(message);
-}
diff --git a/Hawkynt.FileFormats.Archives/README.md b/Hawkynt.FileFormats.Archives/README.md
index 9357f828e..7ab636474 100644
--- a/Hawkynt.FileFormats.Archives/README.md
+++ b/Hawkynt.FileFormats.Archives/README.md
@@ -151,7 +151,7 @@ Every State, Test, Maintenance, Compress/Decompress and Demux/Mux/Remux cell is
| ZAP (Amiga) | `Zap` | `.zap` | WORM | ✅ | wipe | Whole-disk archiver: entries are track_NNN.raw | [Aminet](https://aminet.net/) |
| [ZIP](https://en.wikipedia.org/wiki/ZIP_(file_format)) | `Zip` | `.zip` `.zipx` | R/W | ✅ | compress · repack · shrink · wipe | Store, Deflate, Deflate64, Shrink, Reduce, Implode, BZip2, LZMA, XZ, PPMd, Zstd, AES | [pkware.cachefly.net](https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT) |
| [ZOO](https://en.wikipedia.org/wiki/Zoo_(file_format)) | `Zoo` | `.zoo` | R/W | ✅ | defrag · wipe | Store, LZW, LZH | [Archive Team](http://fileformats.archiveteam.org/wiki/ZOO) |
-| [ZPAQ](https://en.wikipedia.org/wiki/ZPAQ) | `Zpaq` | `.zpaq` | R/W | ✅ | defrag | Reader covers the stored/simple models; ZPAQL virtual-machine execution is not implemented | [mattmahoney.net](http://mattmahoney.net/dc/zpaq.html) |
+| [ZPAQ](https://en.wikipedia.org/wiki/ZPAQ) | `Zpaq` | `.zpaq` | R/W | ✅ | defrag | Journaling (versions, deduplicated fragments) and streaming layouts, methods 0-5 and explicit `x` methods, full ZPAQL; zpaq 7.15 extracts what we write. Encrypted, multi-part and `-index` archives are refused; modify rewrites one version | [ZPAQ Level 2 spec](https://mattmahoney.net/dc/zpaq206.pdf) |
### 🧬 Structured data and registry
@@ -544,7 +544,7 @@ Use it when a .NET process needs to enumerate, extract, test, create, edit or in
-Every public and protected member of all 2571 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md).
+Every public and protected member of all 2564 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.Archives/REFERENCE.md).
@@ -559,7 +559,7 @@ Every public and protected member of all 2571 types, generated from the built as
- WORM is not R/W: creating a valid archive is different from safely editing one, and only descriptors with a proven edit path advertise `CanModify`.
- R/W by rebuild rewrites the container; formats whose listing renames entries (track images, chunked backup images, hash-keyed game archives) can only address entries by the names they list.
-- LZFSE: `bvx1` blocks are read but never written. ZPAQ: no ZPAQL virtual machine. StuffIt X and UMX writers emit the envelope shell only. SFAR: LZX payload extraction is limited. Inno Setup: some versions expose no per-file extraction.
+- LZFSE: `bvx1` blocks are read but never written. ZPAQ: encrypted (`-key`) archives are refused. StuffIt X and UMX writers emit the envelope shell only. SFAR: LZX payload extraction is limited. Inno Setup: some versions expose no per-file extraction.
- OLE2 (DOC / XLS / PPT / MSG / Thumbs.db / MSI) creation produces a valid CFB envelope, not the application's document or database streams.
- RAR and 7z creation target the implemented RAR4/RAR5 and 7z paths, not every historical writer version, and no vendor encoder heuristic is reproduced.
- Media containers are demuxed at the container level; carried codecs are decoded only where the audio package provides them.
diff --git a/Hawkynt.FileFormats.Archives/REFERENCE.md b/Hawkynt.FileFormats.Archives/REFERENCE.md
index 464eb0ba6..e4e44e79e 100644
--- a/Hawkynt.FileFormats.Archives/REFERENCE.md
+++ b/Hawkynt.FileFormats.Archives/REFERENCE.md
@@ -33136,43 +33136,11 @@ Implements `IDisposable`.
### Namespace `FileFormat.Zpaq`
-[`Component`](#component) · [`ComponentType`](#componenttype) · [`ZpaqBuildingBlock`](#zpaqbuildingblock) · [`ZpaqConstants`](#zpaqconstants) · [`ZpaqEntry`](#zpaqentry) · [`ZpaqFormatDescriptor`](#zpaqformatdescriptor) · [`ZpaqRangeCoder`](#zpaqrangecoder) · [`ZpaqRangeDecoder`](#zpaqrangedecoder) · [`ZpaqRangeEncoder`](#zpaqrangeencoder) · [`ZpaqReader`](#zpaqreader) · [`ZpaqWriter`](#zpaqwriter) · [`ZpaqlVm`](#zpaqlvm)
-
-#### `Component`
-
-Represents a single prediction component in the ZPAQL context-mixing model.
-
-| Member | Signature | Summary |
-| --- | --- | --- |
-| `Component` | `Component()` | |
-| `Aux` | `int Aux { get; set; }` | Auxiliary state (match length, mixer weights, etc.). |
-| `Context` | `int Context { get; set; }` | Current hashed context value. |
-| `Memory` | `int[] Memory { get; set; }` | Component memory (CM counters, ICM byte histories, etc.). |
-| `Param1` | `int Param1 { get; set; }` | First parameter (meaning depends on type, e.g. log2 of table size). |
-| `Param2` | `int Param2 { get; set; }` | Second parameter (meaning depends on type). |
-| `Param3` | `int Param3 { get; set; }` | Third parameter (meaning depends on type). |
-| `Prediction` | `int Prediction { get; set; }` | Current prediction (0..65535, probability of next bit being 1). |
-| `Type` | `ComponentType Type { get; set; }` | The component type. |
-
-#### `ComponentType`
-
-Component type identifiers for the ZPAQL context-mixing prediction model.
-
-| Value | Numeric | Summary |
-| --- | --- | --- |
-| `Const` | `0` | Constant prediction (always predicts 50/50). |
-| `Cm` | `1` | Context model — 2^n counters indexed by hashed context. |
-| `Icm` | `2` | Indirect context model — byte history maps to a counter. |
-| `Match` | `3` | Match model — predicts by finding a previous matching context. |
-| `Avg` | `4` | Average of two component predictions. |
-| `Mix2` | `5` | Two-input adaptive mixer. |
-| `Mix` | `6` | N-input adaptive mixer. |
-| `Isse` | `7` | Indirect secondary symbol estimation. |
-| `Sse` | `8` | Direct secondary symbol estimation. |
+[`ZpaqBuildingBlock`](#zpaqbuildingblock) · [`ZpaqEntry`](#zpaqentry) · [`ZpaqFormatDescriptor`](#zpaqformatdescriptor) · [`ZpaqReader`](#zpaqreader) · [`ZpaqWriter`](#zpaqwriter)
#### `ZpaqBuildingBlock`
-Exposes the ZPAQ context-mixing codec — the compression stage that the ZPAQ journaling archiver wraps — as a benchmarkable building block. Prepends a 4-byte LE uncompressed size header for round-trip support.
+ZPAQ compression as a building block: the input coded as a plain ZPAQ stream (locator tag, blocks of one segment, SHA-1 trailers) with zpaq's method 4 — an order-0 ICM, an ISSE chain up to order 6, a match model and a mixer, all described in the block's own ZPAQL header.
Implements `IBuildingBlock`.
@@ -33186,39 +33154,23 @@ Implements `IBuildingBlock`.
| `Compress` | `byte[] Compress(ReadOnlySpan data)` | |
| `Decompress` | `byte[] Decompress(ReadOnlySpan data)` | |
-#### `ZpaqConstants`
-
-Constants for the ZPAQ archive format (level 1 journaling).
-
-| Member | Signature | Summary |
-| --- | --- | --- |
-| `BlockTypeCompressed` | `const byte BlockTypeCompressed` | Block type byte: compressed data block (contains ZPAQL program + compressed payload). |
-| `BlockTypeData` | `const byte BlockTypeData` | Block type byte: data block in a journaling transaction. Each data block holds the (ZPAQL-compressed) payload for one or more files. |
-| `BlockTypeHeader` | `const byte BlockTypeHeader` | Block type byte: comment/filename header block in a journaling transaction. In a level-1 journaling archive this block carries the transaction date, filenames and per-file attributes for one transaction. |
-| `BlockTypeIndex` | `const byte BlockTypeIndex` | Block type byte: hash/index block that closes a journaling transaction. Contains SHA-1 hashes of the uncompressed file data. |
-| `FileTimeSize` | `const int FileTimeSize` | Size of the Windows FILETIME timestamp stored in journaling header blocks (8 bytes). |
-| `Level1` | `const byte Level1` | Compression level byte for ZPAQ level 1 archives. |
-| `Level2` | `const byte Level2` | Compression level byte for ZPAQ level 2 archives. |
-| `MinBlockHeaderSize` | `const int MinBlockHeaderSize` | Minimum number of bytes in a valid block header (3-byte prefix + 1 level byte + 1 type byte = 5 bytes). |
-| `WindowsToUnixEpochTicks` | `const long WindowsToUnixEpochTicks` | Windows FILETIME ticks (100-nanosecond intervals) from the Windows epoch (1601-01-01) to the Unix epoch (1970-01-01), used when converting timestamps. |
-| `BlockPrefix` | `static ReadOnlySpan BlockPrefix { get; }` | The 3-byte block locator prefix present at the start of every ZPAQ block ("zPQ"). |
-
#### `ZpaqEntry`
-Represents a single file entry recorded in a ZPAQ journaling archive.
+A file or directory as recorded in a ZPAQ archive.
| Member | Signature | Summary |
| --- | --- | --- |
-| `CompressedSize` | `long CompressedSize { get; }` | Gets the compressed size of the data block(s) that carry this file's data. This is the total byte count of the raw (still-compressed) data blocks associated with this entry. |
-| `FileName` | `string FileName { get; }` | Gets the filename as stored in the archive. Path separators are always forward slashes; directory entries end with '/'. |
-| `IsDirectory` | `bool IsDirectory { get; }` | Gets a value indicating whether this entry represents a directory. |
-| `LastModified` | `DateTime? LastModified { get; }` | Gets the last-modified timestamp decoded from the journal header, or `null` when the timestamp could not be recovered. |
-| `Size` | `long Size { get; }` | Gets the uncompressed size of the file in bytes as reported by the archive. This value is zero for directory entries and for files whose size could not be determined from the journal (e.g. when the header block is compressed). |
-| `Version` | `int Version { get; }` | Gets the zero-based transaction index in which this entry was recorded. Each append to a ZPAQ archive constitutes a new transaction; the most recent transaction with a given filename defines the file's current state. |
+| `Attributes` | `long Attributes { get; }` | Raw attribute bytes as stored (first byte 'w' = Windows attributes follow, 'u' = Unix mode follows), packed little-endian; 0 when none were stored. |
+| `CompressedSize` | `long CompressedSize { get; }` | Compressed size attributed to this entry: the share of the data blocks it references, proportional to its new (non-deduplicated) fragments. Zero for directories. |
+| `FileName` | `string FileName { get; }` | Path as stored, with forward slashes; directories end with '/'. |
+| `IsDirectory` | `bool IsDirectory { get; }` | Whether this entry is a directory. |
+| `LastModified` | `DateTime? LastModified { get; }` | Last-modified time (UTC) from the archive's decimal date, or `null`. |
+| `Size` | `long Size { get; }` | Uncompressed size in bytes, or -1 when the archive does not record it (streaming format). |
+| `Version` | `int Version { get; }` | 1-based version (transaction) that last wrote this entry. |
#### `ZpaqFormatDescriptor`
-ZPAQ journaling archive with content-defined deduplication and configurable context-mixing compression. References: `http://mattmahoney.net/dc/zpaq.html` — official ZPAQ page and Level 2 format specification (Matt Mahoney)`https://github.com/zpaq/zpaq` — reference implementation`https://en.wikipedia.org/wiki/ZPAQ` — Wikipedia overview
+ZPAQ archive (journaling and streaming layouts) with deduplication and ZPAQL-described context-mixing compression. References: `https://mattmahoney.net/dc/zpaq206.pdf` — ZPAQ Level 2 specification (Matt Mahoney)`https://mattmahoney.net/dc/zpaq.html` — zpaq / libzpaq reference implementation (public domain)
Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`.
@@ -33226,110 +33178,57 @@ Implements `IArchiveCreatable`, `IArchiveDefragmentable`, `IArchiveFormatOperati
| --- | --- | --- |
| `ZpaqFormatDescriptor` | `ZpaqFormatDescriptor()` | |
| `CanPurgeToEmpty` | `bool CanPurgeToEmpty { get; }` | |
-| `Capabilities` | `FormatCapabilities Capabilities { get; }` | Gets the capabilities. |
-| `Category` | `FormatCategory Category { get; }` | Gets the category. |
-| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | Gets the compound extensions. |
-| `DefaultExtension` | `string DefaultExtension { get; }` | Gets the default extension. |
-| `Description` | `string Description { get; }` | Gets the description. |
-| `DisplayName` | `string DisplayName { get; }` | Gets the display name. |
-| `Extensions` | `IReadOnlyList Extensions { get; }` | Gets the extensions. |
-| `Family` | `AlgorithmFamily Family { get; }` | Gets the family. |
-| `Id` | `string Id { get; }` | Gets the id. |
-| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | Gets the magic signatures. |
-| `Methods` | `IReadOnlyList Methods { get; }` | Gets the methods. |
-| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | Gets the tar compression format id. |
-| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Performs the create operation. |
-| `Defragment` | `void Defragment(Stream archive)` | Rebuild-based defrag: extracts then re-creates the ZPAQ archive in listing order. |
-| `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | Rebuild-based defrag: extracts then re-creates the ZPAQ archive per the requested mode. |
-| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | Decodes the supplied input. |
-| `List` | `List List(Stream stream, string password)` | Lists the entries in the supplied container. |
-
-#### `ZpaqRangeCoder`
-
-Shared constants and the safety argument of the ZPAQ binary range coder.
-
-| Member | Signature | Summary |
-| --- | --- | --- |
-| `CarryEdge` | `const uint CarryEdge` | Value at or above which the top byte of `low` may still be bumped by a carry. |
-| `FlushBytes` | `const int FlushBytes` | Number of `low` bytes the encoder pushes out when the message ends. |
-| `MaximumProbability` | `const int MaximumProbability` | Largest probability of a 1 bit the coder accepts. |
-| `MinimumProbability` | `const int MinimumProbability` | Smallest probability of a 1 bit the coder accepts. |
-| `ProbabilityBits` | `const int ProbabilityBits` | Number of fractional bits in a probability, so probabilities span 0..2^16. |
-| `RangeMinimum` | `const uint RangeMinimum` | Lower bound of the normalised range; the coder renormalises below this. |
-| `Split` | `static uint Split(uint range, int probabilityOfOne)` | Computes the split point between the 1 subrange and the 0 subrange. |
-
-#### `ZpaqRangeDecoder`
-
-Carry-propagating binary range decoder, the exact mirror of `ZpaqRangeEncoder`.
-
-| Member | Signature | Summary |
-| --- | --- | --- |
-| `ZpaqRangeDecoder` | `ZpaqRangeDecoder(byte[] input, int offset)` | Creates a decoder reading from the given buffer, priming its code word from the first `FlushBytes` bytes. |
-| `Range` | `uint Range { get; }` | Gets the current range, for tests that assert the coder invariant. |
-| `DecodeBit` | `int DecodeBit(int probabilityOfOne)` | Decodes one bit against the given probability. |
-
-#### `ZpaqRangeEncoder`
-
-Carry-propagating binary range encoder. See `ZpaqRangeCoder` for the invariant that keeps both subranges non-empty.
-
-| Member | Signature | Summary |
-| --- | --- | --- |
-| `ZpaqRangeEncoder` | `ZpaqRangeEncoder(Stream output)` | Creates an encoder writing its bytes to the given stream. |
-| `Range` | `uint Range { get; }` | Gets the current range, for tests that assert the coder invariant. |
-| `EncodeBit` | `void EncodeBit(int bit, int probabilityOfOne)` | Codes one bit against the given probability. |
-| `Flush` | `void Flush()` | Pushes out the remaining bytes of `low`, ending the coded stream. |
+| `Capabilities` | `FormatCapabilities Capabilities { get; }` | |
+| `Category` | `FormatCategory Category { get; }` | |
+| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | |
+| `DefaultExtension` | `string DefaultExtension { get; }` | |
+| `Description` | `string Description { get; }` | |
+| `DisplayName` | `string DisplayName { get; }` | |
+| `Extensions` | `IReadOnlyList Extensions { get; }` | |
+| `Family` | `AlgorithmFamily Family { get; }` | |
+| `Id` | `string Id { get; }` | |
+| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | |
+| `Methods` | `IReadOnlyList Methods { get; }` | |
+| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | |
+| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | |
+| `Defragment` | `void Defragment(Stream archive)` | Rebuild-based defrag: extracts then re-creates the archive as a single version. |
+| `Defragment` | `void Defragment(Stream archive, DefragOptions options)` | Rebuild-based defrag: extracts then re-creates the archive as a single version. |
+| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | |
+| `List` | `List List(Stream stream, string password)` | |
#### `ZpaqReader`
-Reads the journaling index of a ZPAQ archive, exposing the files recorded across all transactions.
+Reads ZPAQ archives in both layouts the format defines: the journaling layout of `zpaq` 6.x/7.x (versioned transactions of `c`/`d`/`h`/`i` blocks with deduplicated, SHA-1 checked fragments) and the plain streaming layout (one file per block or segment run).
Implements `IDisposable`.
| Member | Signature | Summary |
| --- | --- | --- |
-| `ZpaqReader` | `ZpaqReader(Stream stream, bool leaveOpen = false)` | Opens a ZPAQ archive stream and scans its journal. |
-| `Entries` | `IReadOnlyList Entries { get; }` | Gets the entries discovered by scanning the archive journal. The list reflects the last recorded state of each file across all transactions (i.e. later transactions that mention the same filename supersede earlier ones in the final view). |
+| `ZpaqReader` | `ZpaqReader(Stream stream, bool leaveOpen = false)` | Opens a ZPAQ archive and reads its index. |
+| `Entries` | `IReadOnlyList Entries { get; }` | Files and directories present in the latest version, ordered by name. |
+| `Errors` | `int Errors { get; }` | Number of blocks that failed to decode and were skipped while reading the index. |
+| `VersionCount` | `int VersionCount { get; }` | Number of versions (transactions) in the archive. |
| `Dispose` | `void Dispose()` | |
-| `Extract` | `Stream Extract(ZpaqEntry entry)` | Returns the entry's uncompressed bytes as a stream when the data block was STORED (the layout this toolkit's `ZpaqWriter` emits). |
+| `ExtractBytes` | `byte[] ExtractBytes(ZpaqEntry entry)` | Decodes the entry's content, verifying every fragment's (or segment's) SHA-1. |
+| `Extract` | `Stream Extract(ZpaqEntry entry)` | Returns the entry's content as a read-only stream. |
+| `GetEntries` | `IReadOnlyList GetEntries(int version)` | The archive contents as of version `version` (1-based). |
+| `GetVersionDate` | `DateTime? GetVersionDate(int version)` | UTC date of version `version` (1-based), or `null` for a streaming archive. |
#### `ZpaqWriter`
-Writes a ZPAQ level-1 journaling archive.
+Writes a ZPAQ journaling archive holding one version (transaction): the `c` header, the deduplicated and compressed `d` data blocks, their `h` fragment tables and the `i` index, exactly as `zpaq add` lays them out.
Implements `IDisposable`.
| Member | Signature | Summary |
| --- | --- | --- |
-| `ZpaqWriter` | `ZpaqWriter(Stream stream, bool leaveOpen = false)` | Initializes a new ZPAQ writer that writes to the specified stream. |
-| `AddDirectory` | `void AddDirectory(string dirName, DateTime? lastModified = null)` | Adds a directory entry to the archive. |
-| `AddFile` | `void AddFile(string fileName, byte[] data, DateTime? lastModified = null)` | Adds a file entry to the archive with the given name and data. |
+| `ZpaqWriter` | `ZpaqWriter(Stream stream, bool leaveOpen = false, string method = "1")` | Creates a writer that emits the archive to `stream` on dispose. |
+| `Method` | `string Method { get; }` | The zpaq compression method used for the data blocks. |
+| `TransactionDate` | `DateTime? TransactionDate { get; set; }` | Date of the transaction; defaults to the time of disposal. |
+| `AddDirectory` | `void AddDirectory(string dirName, DateTime? lastModified = null)` | Adds a directory entry. |
+| `AddFile` | `void AddFile(string fileName, byte[] data, DateTime? lastModified = null)` | Adds (or replaces) a file. |
| `Dispose` | `void Dispose()` | |
-#### `ZpaqlVm`
-
-ZPAQL virtual machine for ZPAQ compression and decompression.
-
-| Member | Signature | Summary |
-| --- | --- | --- |
-| `ZpaqlVm` | `ZpaqlVm(ReadOnlySpan program)` | Parses a ZPAQL program header from a binary span and constructs the VM. |
-| `ZpaqlVm` | `ZpaqlVm(byte[] hcomp, byte[] pcomp, int hh, int hm, Component[] components = null)` | Creates a ZPAQL VM with the specified program and component configuration. |
-| `A` | `uint A { get; set; }` | Gets or sets register A (32-bit accumulator). |
-| `B` | `uint B { get; set; }` | Gets or sets register B (32-bit, also indexes M[]). |
-| `C` | `uint C { get; set; }` | Gets or sets register C (32-bit, also indexes M[]). |
-| `Components` | `Component[] Components { get; }` | Gets the context-mixing components. |
-| `D` | `uint D { get; set; }` | Gets or sets register D (32-bit, also indexes M[]). |
-| `F` | `bool F { get; set; }` | Gets or sets the flag register (1-bit). |
-| `H` | `uint[] H { get; }` | Gets the H[] context array. H values are set by the HCOMP program and used as context indices for the prediction components. |
-| `M` | `byte[] M { get; }` | Gets the M[] byte array used by the HCOMP/PCOMP programs. |
-| `OutputByte` | `Action OutputByte { get; set; }` | Event invoked when the PCOMP program executes an OUT instruction. |
-| `Predict` | `int Predict()` | Computes a combined prediction from all components. |
-| `Reset` | `void Reset()` | Resets the VM state (registers, arrays, components) without changing the program or component configuration. |
-| `RunHcomp` | `void RunHcomp(int input)` | Executes the HCOMP program with the given input byte. This should be called after each decoded byte to update the context hashes in H[]. |
-| `RunPcomp` | `void RunPcomp(int input)` | Executes the PCOMP (post-processing) program with the given input byte. |
-| `Squash` | `static int Squash(int x)` | Squash function: maps a logistic value (-2048..2047) to a probability (0..65535). squash(x) = 65536 / (1 + exp(-x / 64)). |
-| `Stretch` | `static int Stretch(int p)` | Stretch function: inverse of squash, maps probability (1..65534) to logistic. |
-| `Update` | `void Update(int bit)` | Updates all components after a decoded bit. |
-
### Namespace `FileFormat.ZxSnapshot`
[`ZxSnapshotFormatDescriptor`](#zxsnapshotformatdescriptor)
diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs
index 6c7d4b452..1c9e17266 100644
--- a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs
+++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs
@@ -33,6 +33,7 @@ public static class FirmwareHexWriter {
/// one the reader rendered. An input list with no payload describes an image
/// with no data, which both formats can write.
///
+ /// More than one non-metadata payload is supplied.
public static FirmwareImage ImageFrom(IReadOnlyList inputs, string sourceFormat) {
ArgumentNullException.ThrowIfNull(inputs);
@@ -53,7 +54,9 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st
}
// Any other single input is the payload, whatever it is called: a caller
// converting a .bin into a .hex should not have to rename it first.
- payload ??= input.ReadContent();
+ if (payload is not null)
+ throw new InvalidDataException($"{sourceFormat} represents one logical firmware payload; multiple payload inputs are ambiguous.");
+ payload = input.ReadContent();
}
List<(uint Address, byte[] Data)> segments;
diff --git a/docs/LICENCE-PROVENANCE.md b/docs/LICENCE-PROVENANCE.md
index ca50e4c15..c018e4066 100644
--- a/docs/LICENCE-PROVENANCE.md
+++ b/docs/LICENCE-PROVENANCE.md
@@ -31,7 +31,7 @@ package carries; everything else keeps a `THIRD-PARTY-NOTICE..txt` beside
| Class | Upstream rows | Notes |
|---|---|---|
-| A | 18 | |
+| A | 19 |
| B | 17 | FFmpeg alone covers 38 codec folders |
| C | 3 | specifications: `docs/CVF-CODECS-ON-DISK.md` (rewrite already under way), `docs/IT-SAMPLE-COMPRESSION-SPEC.md`, `docs/FDS-EXPANSION-SOUND-SPEC.md` |
| R | 8 | |
@@ -75,6 +75,7 @@ package carries; everything else keeps a `THIRD-PARTY-NOTICE..txt` beside
| `Compression.Core/Dictionary/Lzbitmap/` | libzbitmap (Ernesto A. Fernández, Corellium) | MIT | `THIRD-PARTY-NOTICE.libzbitmap.txt` |
| `Compression.Core/Dictionary/Csc/` | libcsc (Fu Siyuan) | Unlicense | `THIRD-PARTY-NOTICE.libcsc.txt` |
| `Compression.Core/Dictionary/Arj/ArjFastestDecoder.cs` | XADMaster `XADARJFastestHandle.m` (Dag Ågren, MacPaw), revision `7cb9ee0abbb163f261e4cb74501e15067032319c` | LGPL-2.1-or-later | `Compression.Core/Dictionary/Arj/THIRD-PARTY-NOTICE.XADMaster.txt` |
+| `Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/` | libzpaq / zpaq 7.15 (Matt Mahoney) | public domain (Unlicense) | `Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Zpaq/THIRD-PARTY-NOTICE.libzpaq.txt` |
| `Hawkynt.FileFormats.Archives/FileFormats/FileFormat.DiskDoubler/DiskDoublerReader.cs`, `Hawkynt.FileFormats.Archives/FileFormats/FileFormat.DiskDoubler/DiskDoublerAdDecoder.cs` | XADMaster `XADDiskDoublerParser.m` and `XADDiskDoublerADnHandle.m`, revision `7cb9ee0abbb163f261e4cb74501e15067032319c` | LGPL-2.1-or-later | `Hawkynt.FileFormats.Archives/FileFormats/FileFormat.DiskDoubler/THIRD-PARTY-NOTICE.XADMaster.txt` |
| `Compression.Core/Dictionary/Lzh/Lh2Decoder.cs`, `Compression.Core/Dictionary/Lzh/Lh3Decoder.cs`, `Compression.Core/Dictionary/Lzh/LhaLegacyBitReader.cs` | XADMaster `XADLZHOldHandles.m`, revision `7cb9ee0abbb163f261e4cb74501e15067032319c` | LGPL-2.1-or-later | `Compression.Core/Dictionary/Lzh/THIRD-PARTY-NOTICE.XADMaster.txt` |
| `Compression.Core/Dictionary/Lzh/Lh1Tree.cs`, `Compression.Core/Dictionary/Lzh/Lh1Decoder.cs`, `Compression.Core/Dictionary/Lzh/LzsDecoder.cs`, `Compression.Core/Dictionary/Lzh/Lz5Decoder.cs`, `Compression.Core/Dictionary/Lzh/Pm1Decoder.cs`, `Compression.Core/Dictionary/Lzh/Pm2Decoder.cs`, `Compression.Core/Dictionary/Lzh/PmaHistory.cs`, `Compression.Core/Dictionary/Lzh/LhaBitInput.cs`, `Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Lzh/LhaHeaderParser.cs` | lhasa (Simon Howard), revision `75ed83559f23e9538e0045c62f53f77ab03d03d6` | ISC | `Compression.Core/Dictionary/Lzh/THIRD-PARTY-NOTICE.lhasa.txt`; `Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Lzh/THIRD-PARTY-NOTICE.lhasa.txt` |
diff --git a/docs/catalogue/GAPS.md b/docs/catalogue/GAPS.md
index 171540636..778ded8b3 100644
--- a/docs/catalogue/GAPS.md
+++ b/docs/catalogue/GAPS.md
@@ -176,7 +176,7 @@ running real-world samples. The file named at the end of each row is where the l
| SquashFS | `SquashFs` | 3 | SquashFS v4 only; v1-v3 rejected (SquashFsSuperblock.cs) |
| UDF | `Udf` | 3 | type-1 partition maps only: no VAT/sparable/metadata partitions (UDF 1.50+ media) (UdfReader.cs) |
| Windows PE | `Mz`, `PeResources` | 3 | archive view of resources/overlay, not a full PE section extractor |
-| ZPAQ | `Zpaq` | 3 | reads only the stored blocks our own writer emits; ZPAQL-compressed blocks (every real zpaq level) throw (ZpaqReader.cs) |
+| ZPAQ | `Zpaq` | 3 | encrypted (`-key`), multi-part and `-index` archives are refused; modify rewrites a single version instead of appending one (ZpaqReader.cs, ZpaqWriter.cs) |
| Compact Pro | `CompactPro` | 2 | stored and RLE only; LZ+RLE and LZ+Huffman unsupported (CompactProReader.cs) |
| DiskDoubler | `DiskDoubler` | 2 | single files only (no DDAR/DDA2 archives); methods 0, 1, 6, 9 decoded and checked against unar on real files; 2-5, 7, 8, 10 and delta refused (DiskDoublerReader.cs) |
| EnCase / Expert Witness (E01) | `Ewf` | 2 | single segment only (no .E02+), no L01 logical evidence, no Ex01 (EwfReader.cs) |
@@ -205,10 +205,7 @@ running real-world samples. The file named at the end of each row is where the l
unsupported block states as zeros, and `Vmdk` returns compressed grains raw. `Arj` treats method 4
as stored, and `Lzh` decodes lh2/lh3 with the lh5 decoder. Each of these gives the caller wrong
data instead of an error.
-- **Capability claims against real files.** `Zpaq` declares `CanCreate | CanModify`, but does not
- read an archive written by the real tool. Its round-trip only proves that our writer and reader
- agree with each other. (`FreeArc` now reads FreeArc 0.67's archives and FreeArc accepts ours; see
- `docs/FREEARC-ON-DISK.md`.)
+
- **Stale comments.** The "(not supported)" comments in `ArcConstants.cs` describe methods the
reader does decode.
diff --git a/docs/catalogue/reference-formats.csv b/docs/catalogue/reference-formats.csv
index d168a2e19..4f5e04e63 100644
--- a/docs/catalogue/reference-formats.csv
+++ b/docs/catalogue/reference-formats.csv
@@ -56,7 +56,7 @@ HA,ha,archive,,,,,,,Y,1,https://en.wikipedia.org/wiki/HA_(file_format),Ha,Y,Y,Y,
SQX,sqx,archive,,,,,,,Y,1,https://en.wikipedia.org/wiki/SQX,Sqx,Y,Y,Y,OK,,
UHARC,uha,archive,,,,,R,,Y,2,https://en.wikipedia.org/wiki/UHarc,Uharc,Y,Y,Y,PARTIAL,,stored and LZP only (UharcReader.cs)
FreeArc,arc;cdx,archive,,,,,R,fmt/1096,Y,3,https://en.wikipedia.org/wiki/FreeArc,FreeArc,Y,Y,Y,PARTIAL,,only storing and lzma blocks decode; tornado/ppmd/lzp/grzip/rep/delta/exe and encryption are refused (FreeArcMethods.cs)
-ZPAQ,zpaq,archive,,,,,R,fmt/1097,RW,3,https://mattmahoney.net/dc/zpaq206.pdf,Zpaq,Y,Y,Y,PARTIAL,,reads only the stored blocks our own writer emits; ZPAQL-compressed blocks (every real zpaq level) throw (ZpaqReader.cs)
+ZPAQ,zpaq,archive,,,,,R,fmt/1097,RW,3,https://mattmahoney.net/dc/zpaq206.pdf,Zpaq,Y,Y,Y,PARTIAL,,"journaling and streaming archives at every method are read and zpaq 7.15 extracts what we write; encrypted (-key), multi-part and -index archives are refused, and modify rewrites a single version instead of appending one (ZpaqReader.cs, ZpaqWriter.cs)"
PAQ (paq6/7/8 family),paq6;paq7;paq8;paq8l,archive,,,,,,,Y,1,https://mattmahoney.net/dc/paq.html,Paq8,Y,Y,,PARTIAL,,a single PAQ8 variant (.paq8l); the other PAQ versions are mutually incompatible
KGB Archiver,kgb;kge,archive,,,,,R,,RW,2,,,,,,MISSING,,
PEA (PeaZip),pea,archive,,,,,R,fmt/1095,Y,3,,,,,,MISSING,,https://peazip.github.io/pea-file-format.html