+ Reiser4 plugin profiles, tails, compressed clusters, links, journal replay, tall-tree defrag

.github/workflows/ci.yml

@@ -232,7 +232,7 @@ jobs:
         run: >
           dotnet test Compression.Tests
           --configuration Release --no-build
-          --filter "FullyQualifiedName~Compression.Tests.Reiser4.Reiser4ProfileExternalTests|FullyQualifiedName~Compression.Tests.Reiser4.Reiser4MutationExternalTests|FullyQualifiedName~Compression.Tests.Reiser4.Reiser4NativeTreeExternalTests"
+          --filter "FullyQualifiedName~Compression.Tests.Reiser4.Reiser4ProfileExternalTests|FullyQualifiedName~Compression.Tests.Reiser4.Reiser4MutationExternalTests|FullyQualifiedName~Compression.Tests.Reiser4.Reiser4NativeTreeExternalTests|FullyQualifiedName~Compression.Tests.Reiser4.Reiser4FeatureExternalTests"
           --logger "trx;LogFileName=reiser4-oracle-tests.trx"
 
       # bcachefs-tools' own checker judges every volume this package writes or

Compression.Tests/Compression.Tests.csproj

@@ -67,6 +67,13 @@
                       Exclude="StructuredPseudoArchives\ReferenceVectors\*.py;StructuredPseudoArchives\ReferenceVectors\*.pl;StructuredPseudoArchives\ReferenceVectors\*.ps1;StructuredPseudoArchives\ReferenceVectors\README.md">
       <LogicalName>ReferenceVectors.%(Filename)%(Extension)</LogicalName>
     </EmbeddedResource>
+    <!-- Volumes a Reiser4 kernel wrote under mkfs.reiser4 plugin profiles, with the kernel's read-back;
+         see Reiser4/ReferenceVectors/README.md. -->
+    <EmbeddedResource Include="Reiser4\ReferenceVectors\*.gz;Reiser4\ReferenceVectors\*.manifest">
+      <LogicalName>Reiser4Vectors.%(Filename)%(Extension)</LogicalName>
+    </EmbeddedResource>
+    <!-- The QEMU harness that lets a Reiser4 kernel judge what the writer produces. -->
+    <None Include="Reiser4\KernelOracle\*.sh" CopyToOutputDirectory="PreserveNewest" LinkBase="Reiser4\KernelOracle" />
     <!-- Volumes bcachefs-tools wrote; see BcacheFs/ReferenceVectors/README.md. -->
     <EmbeddedResource Include="BcacheFs\ReferenceVectors\*.gz;BcacheFs\ReferenceVectors\manifest.txt">
       <LogicalName>BcacheFsVectors.%(Filename)%(Extension)</LogicalName>

Compression.Tests/Reiser4/KernelOracle/.gitattributes

@@ -0,0 +1,2 @@
+# Run by bash under Linux or WSL.
+*.sh text eol=lf

Compression.Tests/Reiser4/KernelOracle/kernel-run.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Boots a Reiser4-enabled Linux kernel under QEMU with an image as /dev/vda and runs a shell
+# script inside it; the script's output is printed between ===GUEST-BEGIN=== and ===GUEST-END===.
+#
+#   kernel-run.sh <image> <guest-script> [payload-dir]
+#
+# The kernel is Metztli's Debian build of Linux 5.4.18 with the Reiser4 (SFRN 4.0.2) patch
+# (linux-image-5.4.0-3+reizer4.0.2-cloud-amd64). It is fetched once into $CWB_REISER4_KERNEL
+# (default ~/.cache/cwb/reiser4-kernel) together with Ubuntu's busybox-static for the initramfs.
+# Nothing here is a build input: the tests that use it skip when it cannot run.
+set -e
+K=${CWB_REISER4_KERNEL:-$HOME/.cache/cwb/reiser4-kernel}
+KV=5.4.0-3+reizer4.0.2-cloud-amd64
+DEB_URL='https://sourceforge.net/projects/metztli-reiser4/files/RyzenEpyc_Reiser4-SFRN-4.0.2_Linux-5.4.18-1_for-Debian_Buster/linux-image-5.4.0-3%2Breizer4.0.2-cloud-amd64_5.4.18-1%2Breizer4.0.2_amd64.deb/download'
+
+if [ "$1" = "--check" ]; then
+  [ -f "$K/x/boot/vmlinuz-$KV" ] && [ -x "$K/bb/usr/bin/busybox" ] && command -v qemu-system-x86_64 >/dev/null
+  exit $?
+fi
+if [ "$1" = "--prepare" ]; then
+  mkdir -p "$K"
+  if [ ! -f "$K/x/boot/vmlinuz-$KV" ]; then
+    curl -sfL -A "Wget/1.21" -o "$K/kernel.deb" "$DEB_URL"
+    mkdir -p "$K/x" && dpkg-deb -x "$K/kernel.deb" "$K/x"
+  fi
+  if [ ! -x "$K/bb/usr/bin/busybox" ]; then
+    (cd "$K" && apt-get download busybox-static >/dev/null 2>&1 && mkdir -p bb && dpkg-deb -x busybox-static*.deb bb)
+  fi
+  [ -f "$K/x/boot/vmlinuz-$KV" ] && [ -x "$K/bb/usr/bin/busybox" ] && command -v qemu-system-x86_64 >/dev/null
+  exit $?
+fi
+
+M=$K/x/lib/modules/$KV/kernel
+R=$(mktemp -d)
+trap 'rm -rf "$R" "$R.cpio.gz" "$R.log"' EXIT
+mkdir -p "$R/bin" "$R/mods" "$R/dev" "$R/proc" "$R/sys" "$R/mnt" "$R/payload"
+cp "$K/bb/usr/bin/busybox" "$R/bin/"
+for a in $("$K/bb/usr/bin/busybox" --list); do [ "$a" = busybox ] || ln -s busybox "$R/bin/$a"; done
+for m in drivers/virtio/virtio.ko drivers/virtio/virtio_ring.ko drivers/virtio/virtio_pci.ko drivers/block/virtio_blk.ko \
+         lib/xxhash.ko lib/zstd/zstd_compress.ko lib/zstd/zstd_decompress.ko crypto/crc32c_generic.ko fs/reiser4/reiser4.ko; do
+  cp "$M/$m" "$R/mods/"
+done
+cp "$2" "$R/run.sh"
+[ -n "$3" ] && cp -a "$3"/. "$R/payload/"
+cat > "$R/init" <<'EOF'
+#!/bin/busybox sh
+export PATH=/bin
+mount -t proc proc /proc; mount -t sysfs sys /sys; mount -t devtmpfs dev /dev
+echo "===GUEST-BEGIN==="
+for m in virtio virtio_ring virtio_pci virtio_blk xxhash zstd_compress zstd_decompress crc32c_generic reiser4; do
+  insmod /mods/$m.ko || echo "insmod $m failed"
+done
+sleep 0.3
+sh /run.sh 2>&1
+echo "===GUEST-END rc=$?==="
+dmesg | grep -i -E "reiser4|bug|oops" | grep -v "loading out-of-tree" | sed 's/^/dmesg: /'
+sync
+poweroff -f
+EOF
+chmod +x "$R/init"
+(cd "$R" && find . | cpio -o -H newc 2>/dev/null | gzip -1) > "$R.cpio.gz"
+ACC=-enable-kvm; [ -w /dev/kvm ] || ACC="-accel tcg"
+timeout ${CWB_REISER4_KERNEL_TIMEOUT:-900} qemu-system-x86_64 $ACC -m 1024 -nographic -no-reboot \
+  -kernel "$K/x/boot/vmlinuz-$KV" -initrd "$R.cpio.gz" \
+  -append "console=ttyS0 quiet panic=-1" \
+  -drive file="$1",format=raw,if=virtio,cache=unsafe 2>&1 | tr -d '\r' > "$R.log" || true
+sed -n '/===GUEST-BEGIN===/,$p' "$R.log" | grep -v '^\s*$'
+grep -q '===GUEST-END' "$R.log"

Compression.Tests/Reiser4/KernelOracle/manifest.sh

@@ -0,0 +1,14 @@
+# Guest script: mounts /dev/vda and prints one line per object the kernel resolves, reading every
+# regular file whole:  path|type|mode|uid|gid|size|nlink|mtime|major:minor|md5-or-link-target
+mount -t reiser4 /dev/vda /mnt || { echo MOUNTFAIL; exit 1; }
+cd /mnt
+echo "===MANIFEST==="
+find . | sort | while read p; do
+  t=$(stat -c '%F|%a|%u|%g|%s|%h|%Y|%t:%T' "$p")
+  if [ -L "$p" ]; then echo "$p|$t|$(readlink "$p")"
+  elif [ -f "$p" ]; then echo "$p|$t|$(md5sum < "$p" | cut -d' ' -f1)"
+  else echo "$p|$t|"; fi
+done
+echo "===END-MANIFEST==="
+cd /
+umount /mnt || echo UMOUNTFAIL

Compression.Tests/Reiser4/ReferenceVectors/.gitattributes

@@ -0,0 +1,5 @@
+# Volumes a Reiser4 kernel wrote, gzip-compressed: the bytes are the fixture.
+*.gz -text
+# Manifests and scripts are read by Linux tools and compared byte for byte.
+*.manifest text eol=lf
+*.sh text eol=lf

Compression.Tests/Reiser4/ReferenceVectors/README.md

@@ -0,0 +1,41 @@
+# Reiser4 reference volumes
+
+Volumes a Reiser4 kernel wrote, checked in so the reader and editor tests run without Linux tools.
+`generate.sh` documents how they were made; it is not a build input.
+
+Each volume was formatted by `mkfs.reiser4 -fffy` (reiser4progs 1.2.2, 16 MiB, UUID
+`11111111-2222-3333-4444-555555555555`) with the plugin options below, then filled by `populate.sh`
+inside the kernel — Metztli's Debian build of Linux 5.4.18 with the Reiser4 SFRN 4.0.2 patch
+(`linux-image-5.4.0-3+reizer4.0.2-cloud-amd64`), booted under QEMU by
+`../KernelOracle/kernel-run.sh`. The tree holds files of 0 to 200 705 bytes (random, and a 69 559-byte text that spans two 64 KiB clusters), nested
+and empty directories, names of 35, 46 and 255 bytes, names ending in `.c`, `.o` and `.abc`,
+a short and a 423-byte symlink, a file with three names, a FIFO, character device 1:3, block device
+7:0, a file with a hole, a 300-entry directory, an owner of 1234:5678 with mode 0751, and a
+modification time in 2001.
+
+| File | mkfs.reiser4 options | What it exercises |
+|------|------------------|-------------------|
+| `default` | — | cryptcompress files (lzo1, conv mode, 64 KiB clusters), the kernel turning incompressible ones into extents |
+| `gzip1-force-8k` | `compress=gzip1,compressMode=force,cluster=8K` | raw-deflate clusters with their Adler-32 |
+| `zstd1-latt-16k` | `compress=zstd1,compressMode=latt,cluster=16K` | Zstandard clusters |
+| `lzo1-none-4k` | `compressMode=none,cluster=4K` | uncompressed clusters |
+| `reg40-smart` | `create=reg40` | tails up to four blocks, extents above |
+| `reg40-tails` | `create=reg40,formatting=tails` | tails only |
+| `reg40-node41` | `create=reg40,formatting=extents,node=node41` | checksummed nodes, extents only |
+| `node41` | `node=node41` | checksummed nodes with cryptcompress files |
+| `tea-lexic` | `create=reg40,hash=tea_hash,fibration=lexic_fibre` | TEA-hashed long names, no fibres |
+| `fnv1-dot-o` | `create=reg40,hash=fnv1_hash,fibration=dot_o_fibre` | FNV-1 long names, `.o` fibre |
+| `deg-ext-3` | `create=reg40,hash=deg_hash,fibration=ext_3_fibre` | degenerate hash, three-letter extension fibres |
+| `rupasov` | `create=reg40,hash=rupasov_hash` | read only: `fsck.reiser4` disagrees with the kernel's Rupasov keys |
+| `journal-replay` | `create=reg40` | a commit the kernel had not yet written back: block 19 names transaction 181, block 20 transaction 130 |
+
+`<name>.manifest` is what the kernel read back after unmounting and mounting again: one line per
+object, `path|type|mode|uid|gid|size|nlink|mtime|major:minor|md5-or-link-target`, every regular file
+read whole. For `journal-replay` the kernel replayed the journal at mount, so the manifest is the
+post-replay state. `default` and `node41` list two fewer `many/` files than the others and
+`gzip1-force-8k` one fewer: the kernel itself did not keep them, and the reader agrees with what it
+did keep.
+
+Every volume except `rupasov` passed `fsck.reiser4 --check` (`journal-replay` after the replay that
+check performs). Captured 2026-10-19 on Ubuntu 24.04 under WSL2 (the WSL kernel has no Reiser4
+driver; the QEMU guest is the only writer). Compressed with `gzip -9 -n`.