+ Fossil (Plan 9 file server) reader

Compression.Tests/Compression.Tests.csproj

@@ -150,6 +150,10 @@
     <EmbeddedResource Include="Codecs\G722\ReferenceVectors\*.pcm;Codecs\G722\ReferenceVectors\*.g722">
       <LogicalName>G722Vectors.%(Filename)%(Extension)</LogicalName>
     </EmbeddedResource>
+    <!-- Fossil volume written by plan9port's fossil server; see Fossil/ReferenceVectors/README.md. -->
+    <EmbeddedResource Include="Fossil\ReferenceVectors\*.gz;Fossil\ReferenceVectors\*.manifest">
+      <LogicalName>FossilVectors.%(Filename)%(Extension)</LogicalName>
+    </EmbeddedResource>
     <!-- Zoo archives written by Zoo 2.10; see Zoo/ReferenceVectors/README.md. -->
     <EmbeddedResource Include="Zoo\ReferenceVectors\*.zoo">
       <LogicalName>ZooVectors.%(Filename)%(Extension)</LogicalName>

Compression.Tests/Fossil/FossilTests.cs

@@ -0,0 +1,110 @@
+using System.Buffers.Binary;
+using System.IO.Compression;
+using System.Security.Cryptography;
+using System.Text;
+using FileSystem.Fossil;
+
+namespace Compression.Tests.Fossil;
+
+/// <summary>
+/// Fossil: the reader against a volume plan9port's <c>flfmt</c> formatted and its
+/// <c>fossil</c> server filled over 9P (see <c>Fossil/ReferenceVectors/README.md</c>),
+/// compared with what that server read back; plus refusals.
+/// </summary>
+[TestFixture]
+public class FossilTests {
+
+  private static byte[] Image() {
+    using var res = typeof(FossilTests).Assembly.GetManifestResourceStream("FossilVectors.fossil.img.gz")
+      ?? throw new InvalidOperationException("fossil.img.gz not embedded");
+    using var gz = new GZipStream(res, CompressionMode.Decompress);
+    using var ms = new MemoryStream();
+    gz.CopyTo(ms);
+    return ms.ToArray();
+  }
+
+  private static Dictionary<string, string> Manifest() {
+    using var res = typeof(FossilTests).Assembly.GetManifestResourceStream("FossilVectors.fossil.manifest")
+      ?? throw new InvalidOperationException("fossil.manifest not embedded");
+    using var sr = new StreamReader(res);
+    return sr.ReadToEnd().Split('\n', StringSplitOptions.RemoveEmptyEntries)
+      .Select(l => l.TrimEnd('\r').Split("  ", 2)).ToDictionary(p => p[1], p => p[0]);
+  }
+
+  private static string Sha(byte[] d) => Convert.ToHexStringLower(SHA256.HashData(d));
+
+  [Test]
+  public void GivenServerWrittenVolume_WhenRead_ThenActiveTreeMatchesWhatTheServerReadBack() {
+    var r = new FossilReader(new MemoryStream(Image(), writable: false));
+    var manifest = Manifest();
+    var files = r.Entries.Where(e => !e.IsDirectory && e.Path.StartsWith("active/", StringComparison.Ordinal)).ToDictionary(e => e.Path);
+    Assert.That(files.Keys, Is.EquivalentTo(manifest.Keys));
+    foreach (var (path, sha) in manifest) Assert.That(Sha(r.Extract(files[path])), Is.EqualTo(sha), path);
+  }
+
+  [Test]
+  public void GivenServerWrittenVolume_WhenRead_ThenDirectoriesModesAndOwnersAreThoseTheServerCreated() {
+    var r = new FossilReader(new MemoryStream(Image(), writable: false));
+    var byPath = r.Entries.ToDictionary(e => e.Path);
+    Assert.Multiple(() => {
+      Assert.That(byPath["active/docs"].IsDirectory, Is.True);
+      Assert.That(byPath["active/docs"].Mode & 0x1FF, Is.EqualTo(0x1FD), "d775");
+      Assert.That(byPath["active/empty-dir"].IsDirectory, Is.True);
+      Assert.That(byPath["active/docs/notes.txt"].Mode & 0x1FF, Is.EqualTo(0x180), "600");
+      Assert.That(byPath["active/hello.txt"].Uid, Is.EqualTo("sys"));
+      Assert.That(byPath["active/big.bin"].Source.Depth, Is.EqualTo(2), "900 KB at 4 KiB blocks needs a two-level pointer tree");
+    });
+  }
+
+  [Test]
+  public void GivenSnapshotTakenBeforeARewrite_WhenRead_ThenTheSnapshotKeepsTheOldBytes() {
+    var r = new FossilReader(new MemoryStream(Image(), writable: false));
+    var snap = r.Entries.Where(e => !e.IsDirectory && e.Path.StartsWith("snapshot/", StringComparison.Ordinal) && e.Path.EndsWith("/hello.txt", StringComparison.Ordinal)).ToList();
+    Assert.That(snap, Has.Count.EqualTo(1), string.Join(", ", r.Entries.Select(e => e.Path)));
+    Assert.That(Encoding.ASCII.GetString(r.Extract(snap[0])), Is.EqualTo("hello from fossil\n"));
+    var active = r.Entries.Single(e => e.Path == "active/hello.txt");
+    Assert.That(Encoding.ASCII.GetString(r.Extract(active)), Is.EqualTo("hello again, after the snapshot was taken\n"));
+  }
+
+  [Test]
+  public void GivenVolume_WhenDetectedByMagic_ThenFossil()
+    => Assert.That(Compression.Lib.FormatDetector.DetectByMagic(Image().AsSpan(0, 192 * 1024)).ToString(), Is.EqualTo("Fossil"));
+
+  [Test]
+  public void GivenWrongHeaderMagic_WhenOpened_ThenInvalidData() {
+    var img = Image();
+    img[FossilReader.HeaderOffset] ^= 0xFF;
+    Assert.Throws<InvalidDataException>(() => new FossilReader(new MemoryStream(img)));
+  }
+
+  [Test]
+  public void GivenWrongSuperblockMagic_WhenOpened_ThenInvalidData() {
+    var img = Image();
+    var bs = BinaryPrimitives.ReadUInt16BigEndian(img.AsSpan(FossilReader.HeaderOffset + 6));
+    var super = BinaryPrimitives.ReadUInt32BigEndian(img.AsSpan(FossilReader.HeaderOffset + 8));
+    img[(int)(super * bs)] ^= 0xFF;
+    Assert.Throws<InvalidDataException>(() => new FossilReader(new MemoryStream(img)));
+  }
+
+  [Test]
+  public void GivenTruncatedImage_WhenRead_ThenRefusedNotGuessed() {
+    var img = Image().AsSpan(0, 200 * 1024).ToArray();
+    Assert.Catch(() => {
+      var r = new FossilReader(new MemoryStream(img));
+      foreach (var e in r.Entries) r.Extract(e);
+    });
+  }
+
+  [Test]
+  public void GivenDamagedMetaBlock_WhenRead_ThenRefused() {
+    var img = Image();
+    var magic = new byte[] { 0x56, 0x56, 0xFC, 0x7A };
+    // Corrupt every meta block's magic: no directory can be listed any more.
+    for (var i = img.AsSpan().IndexOf(magic); i >= 0; ) {
+      img[i] = 0;
+      var next = img.AsSpan(i + 1).IndexOf(magic);
+      i = next < 0 ? -1 : i + 1 + next;
+    }
+    Assert.Throws<InvalidDataException>(() => new FossilReader(new MemoryStream(img)));
+  }
+}

Compression.Tests/Fossil/ReferenceVectors/README.md

@@ -0,0 +1,17 @@
+# Fossil reference volume
+
+`fossil.img.gz` is a 16 MiB Fossil volume (4 KiB blocks) that nothing in this repository
+wrote. It is gzipped only to keep the repository small.
+
+| | |
+| --- | --- |
+| Formatted by | `flfmt -b 4096` from plan9port, commit `1f1396913fb5372408f442ed26933f5ae4b399cd`, built on Ubuntu 24.04 (WSL) |
+| Written by | that plan9port's `fossil` server. Its console created the directories and files; plan9port's `9p` client wrote the data; `fsys main snap` took a local snapshot; `hello.txt` was rewritten after it; `sync`/`halt` closed the volume |
+| Recipe | `generate.sh <output-dir>` |
+| Created | 2026-10-23 |
+| Manifest | sha256 of every active file as the same server read it back over 9P |
+
+The volume covers nested and empty directories, an empty file, a 900,000-byte file
+(a two-level pointer tree at 4 KiB blocks), modes 775/755/664/644/600, and a snapshot
+whose `hello.txt` keeps the bytes from before the rewrite. `leaf.dat` comes from
+`/dev/urandom`, so regenerating changes the image and the manifest together.

Compression.Tests/Fossil/ReferenceVectors/fossil.manifest

@@ -0,0 +1,5 @@
+46d7ffce3c46edc763a81f0392aad235724efb76cf34c5007291a25534129c12  active/big.bin
+f54cbcdce6015b8729e230193ea636f06eecca9088e86ce7e525a803cd584e7d  active/docs/deep/leaf.dat
+3ec6319da06f237aca849b1dad73d310556423b4179661dcc25852a118975468  active/docs/notes.txt
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  active/empty
+886eafda3a87ef99b654084301c104b9c098a3495cd21c03a495ba067287b1fe  active/hello.txt

Compression.Tests/Fossil/ReferenceVectors/generate.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Builds fossil.img.gz + fossil.manifest with plan9port (commit 1f13969): flfmt formats a
+# 16 MiB volume with 4 KiB blocks, the fossil server (console on a pty via script(1))
+# creates the tree, files are written over 9P with plan9port's 9p client, a local snapshot
+# is taken, and hello.txt is rewritten after it. The manifest is what that server reads
+# back over 9P: sha256 and path from the Fossil root.
+# Usage: generate.sh <output-dir>   (expects plan9port built in ~/plan9port)
+set -u
+out="${1:?usage: generate.sh <output-dir>}"; mkdir -p "$out"
+export PLAN9=$HOME/plan9port PATH=$PATH:$HOME/plan9port/bin:$HOME/plan9port/bin/fossil
+d=$(mktemp -d); img=$d/fossil.img; port=$((20000 + RANDOM % 20000)); A="tcp!127.0.0.1!$port"
+truncate -s 16M $img
+flfmt -y -b 4096 $img > /dev/null
+cat > $d/cons.txt <<EOF
+fsys main config $img
+fsys main open -AWVP
+fsys main create /active/docs sys sys d775
+fsys main create /active/docs/deep sys sys d755
+fsys main create /active/empty-dir sys sys d755
+fsys main create /active/hello.txt sys sys 664
+fsys main create /active/empty sys sys 644
+fsys main create /active/big.bin sys sys 644
+fsys main create /active/docs/notes.txt sys sys 600
+fsys main create /active/docs/deep/leaf.dat sys sys 644
+listen $A
+EOF
+mkfifo $d/con
+(script -qfc "fossil -t -c '. $d/cons.txt'" /dev/null < $d/con > $d/fossil.log 2>&1 &)
+exec 3> $d/con
+for i in $(seq 1 30); do 9p -a "$A" ls / >/dev/null 2>&1 && break; sleep 0.5; done
+printf 'hello from fossil\n' | 9p -a "$A" write /hello.txt
+yes "fossil pointer tree " | head -c 900000 | 9p -a "$A" write /big.bin   # 900,000 bytes: 220 blocks > 204 pointers per 4 KiB block, so depth 2
+printf 'private notes\n' | 9p -a "$A" write /docs/notes.txt
+head -c 5000 /dev/urandom | 9p -a "$A" write /docs/deep/leaf.dat
+trap '' PIPE
+echo "fsys main snap" >&3; sleep 3
+printf 'hello again, after the snapshot was taken\n' | 9p -a "$A" write /hello.txt
+mkdir -p $d/x/active/docs/deep
+for f in hello.txt empty big.bin docs/notes.txt docs/deep/leaf.dat; do 9p -a "$A" read /$f > $d/x/active/$f; done
+{ echo "fsys main sync"; sleep 2; echo "fsys main halt"; } >&3 2>/dev/null; sleep 3
+exec 3>&- 2>/dev/null
+pkill -f "$d/cons.txt" 2>/dev/null; sleep 2
+(cd $d/x && find . -type f | sort | sed 's#^\./##' | while read -r f; do echo "$(sha256sum < "$f" | cut -d' ' -f1)  $f"; done) > "$out/fossil.manifest"
+gzip -9 -n -c $img > "$out/fossil.img.gz"
+ls -la "$out"; cat "$out/fossil.manifest"

Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Fossil/FossilFormatDescriptor.cs

@@ -0,0 +1,77 @@
+#pragma warning disable CS1591
+using Compression.Registry;
+using Compression.Registry.Streaming;
+using static Compression.Registry.FormatHelpers;
+
+namespace FileSystem.Fossil;
+
+/// <summary>
+/// Fossil — the Plan 9 file server's on-disk format (Quinlan, McKie, Cox, 2002), as plan9port's
+/// <c>flfmt</c> and <c>fossil</c> write it. Header magic <c>0x3776ae89</c> at 128 KiB.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Read-only. The whole local tree is listed from the Fossil root, so the live file system
+/// is <c>active/…</c> and local snapshots, if any, are <c>snapshot/…</c>. Blocks already
+/// archived to Venti are refused, because there is no Venti here to fetch them from.
+/// </para>
+/// <para>
+/// Proof: volumes made by plan9port's <c>flfmt</c> and filled by its <c>fossil</c> server
+/// over 9P read byte-identical to what that server returns. Sourcing: rung 1 — converted
+/// from plan9port's MIT-licensed fossil sources (THIRD-PARTY-NOTICE.Fossil.txt). No writer:
+/// a writer must keep the label partition, epochs and tags consistent, and is not yet
+/// proven against <c>fossil</c>.
+/// </para>
+/// </remarks>
+public sealed class FossilFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations {
+  public string Id => "Fossil";
+  public string DisplayName => "Fossil (Plan 9)";
+  public FormatCategory Category => FormatCategory.Archive;
+  public FormatCapabilities Capabilities =>
+    FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest |
+    FormatCapabilities.SupportsMultipleEntries | FormatCapabilities.SupportsDirectories;
+  public string DefaultExtension => ".fossil";
+  public IReadOnlyList<string> Extensions => [".fossil"];
+  public IReadOnlyList<string> CompoundExtensions => [];
+  public IReadOnlyList<MagicSignature> MagicSignatures => [
+    new([0x37, 0x76, 0xAE, 0x89, 0x00, 0x01], Offset: FossilReader.HeaderOffset, Confidence: 0.90),
+  ];
+  public IReadOnlyList<FormatMethodInfo> Methods => [new("stored", "Stored")];
+  public string? TarCompressionFormatId => null;
+  public AlgorithmFamily Family => AlgorithmFamily.Archive;
+  public string Description => "Plan 9 Fossil file server volume (local blocks); read-only, Venti-archived blocks refused.";
+
+  public List<ArchiveEntryInfo> List(Stream stream, string? password) {
+    var r = new FossilReader(stream);
+    return r.Entries.Select((e, i) => new ArchiveEntryInfo(i, e.Path, e.Size, e.Size, "Stored", e.IsDirectory, false,
+      DateTime.UnixEpoch.AddSeconds(e.MTime))).ToList();
+  }
+
+  public void Extract(Stream stream, string outputDir, string? password, string[]? files) {
+    var r = new FossilReader(stream);
+    foreach (var e in r.Entries) {
+      if (files != null && !MatchesFilter(e.Path, files)) continue;
+      if (e.IsDirectory) {
+        Directory.CreateDirectory(Path.Combine(outputDir, e.Path.Replace('/', Path.DirectorySeparatorChar)));
+        continue;
+      }
+      WriteFile(outputDir, e.Path, r.Extract(e));
+    }
+  }
+
+  public Stream OpenEntry(Stream archive, string entryName, string? password) {
+    var bytes = this.ExtractEntryToMemory(archive, entryName, password);
+    return new BoundedEntryStream(new MemoryStream(bytes, writable: false), bytes.Length, leaveOpen: false);
+  }
+
+  public byte[] ExtractEntryToMemory(Stream archive, string entryName, string? password) {
+    ArgumentNullException.ThrowIfNull(archive);
+    ArgumentNullException.ThrowIfNull(entryName);
+    if (archive.CanSeek) archive.Position = 0;
+    var r = new FossilReader(archive);
+    var name = entryName.Replace('\\', '/').TrimStart('/');
+    var e = r.Entries.FirstOrDefault(x => !x.IsDirectory && x.Path == name)
+      ?? throw new FileNotFoundException($"Fossil volume has no file '{entryName}'.", entryName);
+    return r.Extract(e);
+  }
+}