diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60a0a65df..d24cac9b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,6 +174,14 @@ jobs: # reiser4progs owns Reiser4: fsck.reiser4 and debugfs.reiser4 judge every volume the # Reiser4 writer creates or edits, in the gate below. sudo apt-get install -y reiser4progs + # mkbootimg is AOSP's own boot image tool pair: the Android boot image writer must + # reproduce mkbootimg's bytes, and unpack_bootimg must read back what it writes. + sudo apt-get install -y mkbootimg + # avbtool judges vbmeta images; no distribution packages it, so the script is fetched + # at a pinned AOSP revision. + curl -sSfL "https://android.googlesource.com/platform/external/avb/+/c5066a96caa7bf4150c0a8cc8cc14ab81733fdc7/avbtool.py?format=TEXT" \ + | base64 -d > "$RUNNER_TEMP/avbtool.py" \ + && echo "CWB_AVBTOOL=$RUNNER_TEMP/avbtool.py" >> "$GITHUB_ENV" || true sudo apt-get install -y "linux-modules-extra-$(uname -r)" || true sudo modprobe gfs2 || true diff --git a/Compression.Tests/AndroidBoot/AndroidBootImageMkbootimgOracleTests.cs b/Compression.Tests/AndroidBoot/AndroidBootImageMkbootimgOracleTests.cs new file mode 100644 index 000000000..05b4c8c80 --- /dev/null +++ b/Compression.Tests/AndroidBoot/AndroidBootImageMkbootimgOracleTests.cs @@ -0,0 +1,212 @@ +#pragma warning disable CS1591 +using System.Diagnostics; +using System.Text; +using Win32Exception = System.ComponentModel.Win32Exception; + +namespace Compression.Tests.AndroidBoot; + +/// +/// The Android boot image descriptor judged by AOSP's own tools: mkbootimg builds an image, +/// the descriptor reads it, and writing the components it read back out must reproduce +/// mkbootimg's bytes exactly; unpack_bootimg must extract from our images exactly the +/// components we put in. +/// +/// +/// The tools are found as mkbootimg / unpack_bootimg on the PATH (Debian/Ubuntu package +/// mkbootimg), or through CWB_MKBOOTIMG / CWB_UNPACK_BOOTIMG naming either an +/// executable or the .py script from https://android.googlesource.com/platform/system/tools/mkbootimg. +/// Without them the tests are ignored. +/// +[TestFixture] +[Category("ArchiveExternal")] +public sealed class AndroidBootImageMkbootimgOracleTests { + + private string _dir = null!; + + [SetUp] + public void SetUp() { + this._dir = Path.Combine(Path.GetTempPath(), "cwb_mkbootimg_" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(this._dir); + } + + [TearDown] + public void TearDown() { + try { Directory.Delete(this._dir, true); } catch { /* best effort */ } + } + + private static readonly Lazy Mkbootimg = new(() => FindTool("CWB_MKBOOTIMG", "mkbootimg")); + private static readonly Lazy UnpackBootimg = new(() => FindTool("CWB_UNPACK_BOOTIMG", "unpack_bootimg")); + + /// Runs a script with its own directory importable, which an embedded Python's ._pth would otherwise prevent. + internal const string RunScript = "import os, runpy, sys; sys.argv = sys.argv[1:]; sys.path.insert(0, os.path.dirname(os.path.abspath(sys.argv[0]))); runpy.run_path(sys.argv[0], run_name='__main__')"; + + private static string[]? FindTool(string variable, string name) { + var configured = Environment.GetEnvironmentVariable(variable); + var candidates = new List(); + if (!string.IsNullOrWhiteSpace(configured)) { + if (configured.EndsWith(".py", StringComparison.OrdinalIgnoreCase)) { + candidates.Add(["python3", "-c", RunScript, configured]); + candidates.Add(["python", "-c", RunScript, configured]); + } else candidates.Add([configured]); + } + candidates.Add([name]); + foreach (var c in candidates) + try { + if (Run(c, ["--help"]).ExitCode == 0) return c; + } catch (Win32Exception) { + // not there + } + return null; + } + + private static (int ExitCode, string StdOut, string StdErr) Run(string[] tool, IEnumerable arguments) { + var start = new ProcessStartInfo(tool[0]) { + RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true, + }; + foreach (var a in tool.Skip(1).Concat(arguments)) start.ArgumentList.Add(a); + using var process = Process.Start(start)!; + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEnd(); + process.WaitForExit(); + return (process.ExitCode, stdout.Result, stderr); + } + + private static string[] Require(Lazy tool, string name) { + if (tool.Value is null) Assert.Ignore($"{name} not found (apt install mkbootimg, or set CWB_MKBOOTIMG / CWB_UNPACK_BOOTIMG)."); + return tool.Value!; + } + + private string Save(string name, byte[] data) { + var path = Path.Combine(this._dir, name); + File.WriteAllBytes(path, data); + return path; + } + + private byte[] RunMkbootimg(IEnumerable arguments, string outputFlag = "--output") { + var output = Path.Combine(this._dir, "reference.img"); + var (code, _, stderr) = Run(Require(Mkbootimg, "mkbootimg"), arguments.Concat([outputFlag, output])); + Assert.That(code, Is.Zero, $"mkbootimg failed:\n{stderr}"); + return File.ReadAllBytes(output); + } + + private static byte[] B(int length, int seed) => AndroidBootImageTests.Bytes(length, seed); + + private static IEnumerable BootCases() { + yield return new TestCaseData(0, 2048, false).SetName("GivenMkbootimgV0_WhenReadAndRewritten_ThenTheBytesAreIdentical"); + yield return new TestCaseData(0, 4096, true).SetName("GivenMkbootimgV0WithSecondAndLongCmdline_WhenReadAndRewritten_ThenTheBytesAreIdentical"); + yield return new TestCaseData(1, 2048, true).SetName("GivenMkbootimgV1WithRecoveryDtbo_WhenReadAndRewritten_ThenTheBytesAreIdentical"); + yield return new TestCaseData(2, 4096, true).SetName("GivenMkbootimgV2WithDtb_WhenReadAndRewritten_ThenTheBytesAreIdentical"); + yield return new TestCaseData(3, 4096, false).SetName("GivenMkbootimgV3_WhenReadAndRewritten_ThenTheBytesAreIdentical"); + yield return new TestCaseData(4, 4096, false).SetName("GivenMkbootimgV4_WhenReadAndRewritten_ThenTheBytesAreIdentical"); + } + + [TestCaseSource(nameof(BootCases))] + public void MkbootimgBootImage_ReadsAndRewritesIdentically(int version, int page, bool extras) { + var kernel = B(12345, 1); + var ramdisk = B(6789, 2); + var args = new List { + "--header_version", version.ToString(System.Globalization.CultureInfo.InvariantCulture), + "--kernel", this.Save("k", kernel), "--ramdisk", this.Save("r", ramdisk), + "--os_version", "13.0.2", "--os_patch_level", "2024-05", + "--cmdline", extras ? "console=ttyMSM0,115200n8 " + new string('x', 600) : "console=ttyS0 quiet", + }; + var expected = new Dictionary { ["kernel"] = kernel, ["ramdisk"] = ramdisk }; + if (version < 3) { + args.AddRange(["--pagesize", page.ToString(System.Globalization.CultureInfo.InvariantCulture), "--base", "0x80000000", "--board", "cwb-board"]); + if (extras && version < 3) { + var second = B(999, 3); + args.AddRange(["--second", this.Save("s", second)]); + expected["second"] = second; + } + if (extras && version >= 1) { + var dtbo = B(2222, 4); + args.AddRange(["--recovery_dtbo", this.Save("o", dtbo)]); + expected["recovery_dtbo"] = dtbo; + } + if (version == 2) { + var dtb = B(3333, 5); + args.AddRange(["--dtb", this.Save("d", dtb)]); + expected["dtb"] = dtb; + } + } + var reference = this.RunMkbootimg(args); + + var ours = AndroidBootImageTests.Extract(reference); + foreach (var (name, data) in expected) + Assert.That(ours[name], Is.EqualTo(data), $"component '{name}' read differently from what mkbootimg stored"); + Assert.That(ours.Keys.Except(["metadata.ini"]), Is.EquivalentTo(expected.Keys)); + Assert.That(Encoding.UTF8.GetString(ours["metadata.ini"]), Does.Contain("os_version = 13.0.2").And.Contain("os_patch_level = 2024-05")); + if (version < 3) Assert.That(Encoding.UTF8.GetString(ours["metadata.ini"]), Does.Contain("image_id_valid = true")); + + Assert.That(AndroidBootImageTests.RebuildFromExtraction(reference), Is.EqualTo(reference), + "the components and header read from mkbootimg's image did not write back to the same bytes"); + } + + [TestCase(3), TestCase(4)] + public void GivenMkbootimgVendorBoot_WhenReadAndRewritten_ThenTheBytesAreIdentical(int version) { + var dtb = B(4321, 6); + var args = new List { + "--header_version", version.ToString(System.Globalization.CultureInfo.InvariantCulture), + "--pagesize", "4096", "--base", "0x40000000", "--board", "vendorbrd", + "--vendor_cmdline", "androidboot.console=ttyS0 loglevel=4", "--dtb", this.Save("d", dtb), + }; + var expected = new Dictionary { ["dtb"] = dtb }; + var platform = B(7000, 7); + args.AddRange(["--vendor_ramdisk", this.Save("vr", platform)]); + if (version == 3) expected["vendor_ramdisk"] = platform; + else { + var dlkm = B(1234, 8); + var bootconfig = Encoding.ASCII.GetBytes("androidboot.hardware=cwb\n"); + args.AddRange(["--vendor_bootconfig", this.Save("bc", bootconfig), + "--ramdisk_type", "dlkm", "--ramdisk_name", "dlkm", "--board_id0", "0xC0FFEE", "--vendor_ramdisk_fragment", this.Save("dl", dlkm)]); + expected["vendor_ramdisk00"] = platform; + expected["vendor_ramdisk01"] = dlkm; + expected["bootconfig"] = bootconfig; + } + var reference = this.RunMkbootimg(args, "--vendor_boot"); + + var ours = AndroidBootImageTests.Extract(reference); + foreach (var (name, data) in expected) + Assert.That(ours[name], Is.EqualTo(data), $"component '{name}' read differently from what mkbootimg stored"); + if (version == 4) + Assert.That(Encoding.UTF8.GetString(ours["metadata.ini"]), Does.Contain("[vendor_ramdisk01]\ntype = 3\nname = dlkm\nboard_id = 0x00C0FFEE,")); + + Assert.That(AndroidBootImageTests.RebuildFromExtraction(reference), Is.EqualTo(reference)); + } + + [Test] + public void GivenAnImageBuiltByHand_WhenMkbootimgBuildsTheSame_ThenTheBytesAreIdentical() { + var kernel = B(20000, 9); + var ramdisk = B(3000, 10); + var dtb = B(500, 11); + var reference = this.RunMkbootimg([ + "--header_version", "2", "--kernel", this.Save("k", kernel), "--ramdisk", this.Save("r", ramdisk), "--dtb", this.Save("d", dtb), + "--pagesize", "2048", "--cmdline", "root=/dev/ram0", "--board", "hand", "--os_version", "10", "--os_patch_level", "2020-01", + ]); + var ours = AndroidBootImageTests.Create(("kernel", kernel), ("ramdisk", ramdisk), ("dtb", dtb), ("metadata.ini", AndroidBootImageTests.Ini( + "[android_boot]\nheader_version = 2\ncmdline = root=/dev/ram0\nboard = hand\nos_version = 10.0.0\nos_patch_level = 2020-01\n"))); + Assert.That(ours, Is.EqualTo(reference)); + } + + [Test] + public void GivenAnEditedImage_WhenUnpackBootimgReadsIt_ThenItExtractsTheNewComponents() { + var unpack = Require(UnpackBootimg, "unpack_bootimg"); + var image = AndroidBootImageTests.Create(("kernel", B(5000, 12)), ("ramdisk", B(800, 13)), ("recovery_dtbo", B(900, 14)), + ("metadata.ini", AndroidBootImageTests.Ini("[android_boot]\nheader_version = 1\ncmdline = a=b\n"))); + var newRamdisk = B(4100, 15); + using var ms = new MemoryStream(); + ms.Write(image); + new FileFormat.AndroidBootImage.AndroidBootImageFormatDescriptor().Add(ms, [Compression.Registry.ArchiveInputInfo.InMemory("ramdisk", newRamdisk)]); + + var path = this.Save("edited.img", ms.ToArray()); + var outDir = Path.Combine(this._dir, "out"); + var (code, stdout, stderr) = Run(unpack, ["--boot_img", path, "--out", outDir]); + Assert.That(code, Is.Zero, $"unpack_bootimg rejected our image:\n{stderr}"); + Assert.Multiple(() => { + Assert.That(File.ReadAllBytes(Path.Combine(outDir, "kernel")), Is.EqualTo(B(5000, 12))); + Assert.That(File.ReadAllBytes(Path.Combine(outDir, "ramdisk")), Is.EqualTo(newRamdisk)); + Assert.That(File.ReadAllBytes(Path.Combine(outDir, "recovery_dtbo")), Is.EqualTo(B(900, 14))); + Assert.That(stdout, Does.Contain("command line args: a=b")); + }); + } +} diff --git a/Compression.Tests/AndroidBoot/AndroidBootImageTests.cs b/Compression.Tests/AndroidBoot/AndroidBootImageTests.cs new file mode 100644 index 000000000..df82a6e7e --- /dev/null +++ b/Compression.Tests/AndroidBoot/AndroidBootImageTests.cs @@ -0,0 +1,391 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Security.Cryptography; +using System.Text; +using Compression.Registry; +using FileFormat.AndroidBootImage; + +namespace Compression.Tests.AndroidBoot; + +/// +/// Android boot / vendor_boot images: the layout, header fields and image id the descriptor +/// writes, how it reads its own and hand-built images back, and how edits behave — including +/// the refusals for signed and non-canonical images. +/// +[TestFixture] +public sealed class AndroidBootImageTests { + + private static readonly AndroidBootImageFormatDescriptor Descriptor = new(); + + internal static byte[] Bytes(int length, int seed) { + var data = new byte[length]; + new Random(seed).NextBytes(data); + return data; + } + + internal static byte[] Create(params (string Name, byte[] Data)[] inputs) { + using var ms = new MemoryStream(); + Descriptor.Create(ms, [.. inputs.Select(i => ArchiveInputInfo.InMemory(i.Name, i.Data))], new FormatCreateOptions()); + return ms.ToArray(); + } + + internal static byte[] Ini(string text) => Encoding.UTF8.GetBytes(text.Replace("\r", "")); + + internal static Dictionary Extract(byte[] image) { + var dir = Path.Combine(Path.GetTempPath(), "cwb_abi_" + Guid.NewGuid().ToString("N")[..10]); + try { + using (var ms = new MemoryStream(image)) Descriptor.Extract(ms, dir, null, null); + return Directory.GetFiles(dir).ToDictionary(f => Path.GetFileName(f), File.ReadAllBytes, StringComparer.Ordinal); + } finally { + try { Directory.Delete(dir, true); } catch { /* best effort */ } + } + } + + internal static byte[] RebuildFromExtraction(byte[] image) + => Create([.. Extract(image).Select(kv => (kv.Key, kv.Value))]); + + private static uint U32(byte[] d, int o) => BinaryPrimitives.ReadUInt32LittleEndian(d.AsSpan(o)); + + // ── layout ─────────────────────────────────────────────────────────── + + [Test, Category("HappyPath")] + public void GivenKernelAndRamdisk_WhenCreatedWithDefaults_ThenTheV0HeaderCarriesMkbootimgDefaults() { + var kernel = Bytes(3000, 1); + var ramdisk = Bytes(100, 2); + var image = Create(("kernel", kernel), ("ramdisk", ramdisk)); + + Assert.Multiple(() => { + Assert.That(Encoding.ASCII.GetString(image, 0, 8), Is.EqualTo("ANDROID!")); + Assert.That(U32(image, 8), Is.EqualTo(3000)); + Assert.That(U32(image, 12), Is.EqualTo(0x10008000u), "kernel address = base + kernel_offset"); + Assert.That(U32(image, 16), Is.EqualTo(100)); + Assert.That(U32(image, 20), Is.EqualTo(0x11000000u), "ramdisk address = base + ramdisk_offset"); + Assert.That(U32(image, 28), Is.Zero, "no second stage, so its address is zero as mkbootimg writes it"); + Assert.That(U32(image, 32), Is.EqualTo(0x10000100u)); + Assert.That(U32(image, 36), Is.EqualTo(2048)); + Assert.That(U32(image, 40), Is.Zero); + // header page + 2 kernel pages + 1 ramdisk page + Assert.That(image.Length, Is.EqualTo(2048 * 4)); + Assert.That(image.AsSpan(2048, 3000).ToArray(), Is.EqualTo(kernel)); + Assert.That(image.AsSpan(2048 * 3, 100).ToArray(), Is.EqualTo(ramdisk)); + }); + } + + [Test, Category("HappyPath")] + public void GivenComponents_WhenCreated_ThenTheImageIdIsTheMkbootimgSha1() { + var kernel = Bytes(10, 3); + var ramdisk = Bytes(20, 4); + var image = Create(("kernel", kernel), ("ramdisk", ramdisk)); + + using var sha = IncrementalHash.CreateHash(HashAlgorithmName.SHA1); + foreach (var part in new[] { kernel, ramdisk, [] }) { + sha.AppendData(part); + sha.AppendData(BitConverter.GetBytes((uint)part.Length)); + } + var expected = new byte[32]; + sha.GetHashAndReset().CopyTo(expected, 0); + Assert.That(image.AsSpan(576, 32).ToArray(), Is.EqualTo(expected)); + } + + [TestCase(2048), TestCase(4096)] + [Category("Boundary")] + public void GivenAKernelOfExactlyOnePage_WhenCreated_ThenNoPaddingPageIsAdded(int page) { + var image = Create(("kernel", Bytes(page, 5)), ("metadata.ini", Ini($"[android_boot]\npage_size = {page}\n"))); + Assert.That(image.Length, Is.EqualTo(2 * page)); + } + + [Test, Category("Boundary")] + public void GivenAKernelOfOnePagePlusOneByte_WhenCreated_ThenItTakesTwoPages() { + var image = Create(("kernel", Bytes(2049, 6))); + Assert.That(image.Length, Is.EqualTo(3 * 2048)); + } + + [Test, Category("Boundary")] + public void GivenNoInputs_WhenCreated_ThenTheImageIsOneHeaderPageAndListsOnlyMetadata() { + var image = Create(); + Assert.That(image.Length, Is.EqualTo(2048)); + using var ms = new MemoryStream(image); + Assert.That(Descriptor.List(ms, null).Select(e => e.Name), Is.EqualTo(new[] { "metadata.ini" })); + } + + [TestCase(0), TestCase(1), TestCase(2), TestCase(3), TestCase(4)] + [Category("RoundTrip")] + public void GivenEachBootHeaderVersion_WhenExtractedAndRecreated_ThenTheBytesAreIdentical(int version) { + var inputs = new List<(string, byte[])> { + ("kernel", Bytes(5000, 10 + version)), ("ramdisk", Bytes(777, 20 + version)), + ("metadata.ini", Ini($"[android_boot]\nheader_version = {version}\ncmdline = console=ttyMSM0 androidboot.hardware=qcom\nos_version = 12.1.0\nos_patch_level = 2023-07\n" + + (version < 3 ? "board = cwbboard\n" : ""))), + }; + if (version < 3) inputs.Add(("second", Bytes(300, 30))); + if (version is 1 or 2) inputs.Add(("recovery_dtbo", Bytes(1500, 31))); + if (version == 2) inputs.Add(("dtb", Bytes(2100, 32))); + if (version == 4) inputs.Add(("boot_signature", Bytes(4096, 33))); + var image = Create([.. inputs]); + + Assert.That(RebuildFromExtraction(image), Is.EqualTo(image)); + } + + [TestCase(3), TestCase(4)] + [Category("RoundTrip")] + public void GivenEachVendorBootVersion_WhenExtractedAndRecreated_ThenTheBytesAreIdentical(int version) { + var inputs = new List<(string, byte[])> { + ("dtb", Bytes(3000, 40)), + ("metadata.ini", Ini($"[android_vendor_boot]\nheader_version = {version}\npage_size = 4096\ncmdline = androidboot.console=ttyS0\nboard = vendorbrd\n")), + }; + if (version == 3) inputs.Add(("vendor_ramdisk", Bytes(9000, 41))); + else { + inputs.Add(("vendor_ramdisk00", Bytes(1000, 42))); + inputs.Add(("vendor_ramdisk01", Bytes(2001, 43))); + inputs.Add(("bootconfig", Encoding.ASCII.GetBytes("androidboot.hardware=cwb\n"))); + } + var image = Create([.. inputs]); + + Assert.That(Encoding.ASCII.GetString(image, 0, 8), Is.EqualTo("VNDRBOOT")); + Assert.That(RebuildFromExtraction(image), Is.EqualTo(image)); + } + + [Test, Category("HappyPath")] + public void GivenAVendorBootV4_WhenListed_ThenFragmentsAndTheirTableFieldsSurface() { + var image = Create(("vendor_ramdisk00", Bytes(10, 1)), ("vendor_ramdisk01", Bytes(20, 2)), + ("metadata.ini", Ini("[android_vendor_boot]\nheader_version = 4\n[vendor_ramdisk01]\ntype = 3\nname = dlkm\nboard_id = 0x1,0x2\n"))); + var files = Extract(image); + var ini = Encoding.UTF8.GetString(files["metadata.ini"]); + Assert.Multiple(() => { + Assert.That(files["vendor_ramdisk00"], Is.EqualTo(Bytes(10, 1))); + Assert.That(files["vendor_ramdisk01"], Is.EqualTo(Bytes(20, 2))); + Assert.That(ini, Does.Contain("[vendor_ramdisk01]\ntype = 3\nname = dlkm\nboard_id = 0x00000001,0x00000002,")); + Assert.That(ini, Does.Contain("[vendor_ramdisk00]\ntype = 1\nname = \n")); + }); + } + + [Test, Category("HappyPath")] + public void GivenAV2CommandLineLongerThan511Bytes_WhenCreated_ThenTheRestSpillsIntoExtraCmdline() { + var cmdline = new string('a', 511) + "TAIL"; + var image = Create(("metadata.ini", Ini($"[android_boot]\ncmdline = {cmdline}\n"))); + Assert.Multiple(() => { + Assert.That(Encoding.ASCII.GetString(image, 64, 511), Is.EqualTo(new string('a', 511))); + Assert.That(image[64 + 511], Is.Zero); + Assert.That(Encoding.ASCII.GetString(image, 608, 4), Is.EqualTo("TAIL")); + }); + } + + [Test] + public void GivenOneFileUnderAnArbitraryName_WhenCreated_ThenItIsTheKernel() { + var payload = Bytes(3000, 11); + var image = Create(("PROBE.BIN", payload)); + var files = Extract(image); + Assert.That(files.Keys, Is.EquivalentTo(new[] { "metadata.ini", "kernel" })); + Assert.That(files["kernel"], Is.EqualTo(payload)); + Assert.That(image, Is.EqualTo(Create(("kernel", payload))), "the same image mkbootimg --kernel would make"); + } + + [Test] + public void GivenOneArbitraryFileBesideARamdisk_WhenCreated_ThenItIsTheKernelAndTheRamdiskStays() { + var files = Extract(Create(("vmlinuz", [1, 2, 3]), ("ramdisk", [4, 5]))); + Assert.That(files["kernel"], Is.EqualTo(new byte[] { 1, 2, 3 })); + Assert.That(files["ramdisk"], Is.EqualTo(new byte[] { 4, 5 })); + } + + // ── refusals on create ─────────────────────────────────────────────── + + [Test, Category("Exception")] + public void GivenTwoArbitraryFileNames_WhenCreated_ThenTheyAreRefused() + => Assert.That(() => Create(("a.bin", [1]), ("b.bin", [2])), Throws.ArgumentException); + + [Test, Category("Exception")] + public void GivenAKernelAndAnArbitraryFileName_WhenCreated_ThenTheArbitraryOneIsRefused() + => Assert.That(() => Create(("kernel", [1]), ("readme.txt", [2])), Throws.ArgumentException); + + [Test, Category("Exception")] + public void GivenOnlyTheWholeImageView_WhenCreated_ThenItIsRefusedRatherThanTakenAsTheKernel() + => Assert.That(() => Create(("FULL.img", [1])), Throws.ArgumentException); + + [Test, Category("Exception")] + public void GivenASecondStageForAV3Image_WhenCreated_ThenItIsRefused() + => Assert.That(() => Create(("second", [1]), ("metadata.ini", Ini("[android_boot]\nheader_version = 3\n"))), Throws.ArgumentException); + + [Test, Category("Exception")] + public void GivenAnOversizedBoardName_WhenCreated_ThenItIsRefused() + => Assert.That(() => Create(("metadata.ini", Ini("[android_boot]\nboard = 0123456789abcdef\n"))), Throws.ArgumentException); + + [Test, Category("Exception")] + public void GivenHeaderVersion5_WhenCreated_ThenItIsRefused() + => Assert.That(() => Create(("metadata.ini", Ini("[android_boot]\nheader_version = 5\n"))), Throws.ArgumentException); + + // ── malformed input ────────────────────────────────────────────────── + + [Test, Category("Exception")] + public void GivenATruncatedHeader_WhenListed_ThenItFallsBackToTheWholeImageWithoutThrowing() { + var image = Create(("kernel", Bytes(100, 1)))[..600]; + using var ms = new MemoryStream(image); + var entries = Descriptor.List(ms, null); + var files = Extract(image); + Assert.Multiple(() => { + Assert.That(entries.Select(e => e.Name), Is.EqualTo(new[] { "metadata.ini", "FULL.img" })); + Assert.That(Encoding.UTF8.GetString(files["metadata.ini"]), Does.Contain("parse_status = partial")); + Assert.That(files["FULL.img"], Is.EqualTo(image)); + }); + } + + [Test, Category("Exception")] + public void GivenAKernelSizePastTheEnd_WhenListed_ThenItFallsBackRatherThanReturningShortData() { + var image = Create(("kernel", Bytes(5000, 1))); + BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(8), 1_000_000); + using var ms = new MemoryStream(image); + Assert.That(Descriptor.List(ms, null).Select(e => e.Name), Does.Contain("FULL.img").And.Not.Contain("kernel")); + } + + [TestCase(0u), TestCase(1000u), TestCase(3000u)] + [Category("Exception")] + public void GivenAnInvalidPageSize_WhenParsed_ThenItIsRejected(uint page) { + var image = Create(("kernel", Bytes(10, 1))); + BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(36), page); + Assert.That(() => AndroidBootImage.Parse(image), Throws.TypeOf()); + } + + [Test, Category("Exception")] + public void GivenAnUnknownHeaderVersion_WhenParsed_ThenItIsRejected() { + var image = Create(("kernel", Bytes(10, 1))); + BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(40), 9); + Assert.That(() => AndroidBootImage.Parse(image), Throws.TypeOf()); + } + + // ── edits ──────────────────────────────────────────────────────────── + + [Test, Category("HappyPath")] + public void GivenAnImage_WhenTheKernelIsReplaced_ThenTheResultEqualsAFreshImageWithThatKernel() { + var ini = ("metadata.ini", Ini("[android_boot]\nheader_version = 2\ncmdline = quiet\n")); + var image = Create(("kernel", Bytes(5000, 1)), ("ramdisk", Bytes(4000, 2)), ("dtb", Bytes(100, 3)), ini); + var newKernel = Bytes(9000, 4); + + using var ms = new MemoryStream(); + ms.Write(image); + Descriptor.Add(ms, [ArchiveInputInfo.InMemory("kernel", newKernel)]); + + var expected = Create(("kernel", newKernel), ("ramdisk", Bytes(4000, 2)), ("dtb", Bytes(100, 3)), ini); + Assert.That(ms.ToArray(), Is.EqualTo(expected)); + } + + [Test, Category("HappyPath")] + public void GivenAnImage_WhenTheCmdlineIsEditedThroughMetadata_ThenOnlyThatFieldAndNothingElseChanges() { + var image = Create(("kernel", Bytes(100, 1)), ("metadata.ini", Ini("[android_boot]\ncmdline = old\nboard = keep\n"))); + using var ms = new MemoryStream(); + ms.Write(image); + var ini = Encoding.UTF8.GetString(Extract(image)["metadata.ini"]).Replace("cmdline = old", "cmdline = new value"); + Descriptor.Add(ms, [ArchiveInputInfo.InMemory("metadata.ini", Encoding.UTF8.GetBytes(ini))]); + + var edited = ms.ToArray(); + Assert.Multiple(() => { + Assert.That(Encoding.ASCII.GetString(edited, 64, 9), Is.EqualTo("new value")); + Assert.That(Encoding.ASCII.GetString(edited, 48, 4), Is.EqualTo("keep")); + Assert.That(edited.AsSpan(2048).ToArray(), Is.EqualTo(image.AsSpan(2048).ToArray())); + }); + } + + [Test, Category("HappyPath")] + public void GivenAnImage_WhenTheRamdiskIsRemoved_ThenItNoLongerListsAndTheKernelIsUntouched() { + var kernel = Bytes(3000, 1); + var image = Create(("kernel", kernel), ("ramdisk", Bytes(10, 2))); + using var ms = new MemoryStream(); + ms.Write(image); + Descriptor.Remove(ms, ["ramdisk"]); + var files = Extract(ms.ToArray()); + Assert.Multiple(() => { + Assert.That(files.Keys, Is.EquivalentTo(new[] { "metadata.ini", "kernel" })); + Assert.That(files["kernel"], Is.EqualTo(kernel)); + }); + } + + [Test, Category("HappyPath")] + public void GivenAVendorBootV4_WhenAFragmentIsRemoved_ThenTheLaterOnesMoveUp() { + var image = Create(("vendor_ramdisk00", Bytes(10, 1)), ("vendor_ramdisk01", Bytes(20, 2)), ("vendor_ramdisk02", Bytes(30, 3)), + ("metadata.ini", Ini("[android_vendor_boot]\nheader_version = 4\n"))); + using var ms = new MemoryStream(); + ms.Write(image); + Descriptor.Remove(ms, ["vendor_ramdisk01"]); + var files = Extract(ms.ToArray()); + Assert.Multiple(() => { + Assert.That(files["vendor_ramdisk00"], Is.EqualTo(Bytes(10, 1))); + Assert.That(files["vendor_ramdisk01"], Is.EqualTo(Bytes(30, 3))); + Assert.That(files.ContainsKey("vendor_ramdisk02"), Is.False); + }); + } + + [Test, Category("HappyPath")] + public void GivenAnImage_WhenPurged_ThenOnlyTheHeaderRemainsAndItStillParses() { + var image = Create(("kernel", Bytes(3000, 1)), ("ramdisk", Bytes(10, 2)), ("metadata.ini", Ini("[android_boot]\nheader_version = 1\nboard = brd\n"))); + using var ms = new MemoryStream(); + ms.Write(image); + Descriptor.Purge(ms); + var purged = ms.ToArray(); + var parsed = AndroidBootImage.Parse(purged); + Assert.Multiple(() => { + Assert.That(purged.Length, Is.EqualTo(2048)); + Assert.That(parsed.HeaderVersion, Is.EqualTo(1)); + Assert.That(FileFormat.FirmwareCommon.FirmwareArchive.CString(parsed.Board), Is.EqualTo("brd")); + Assert.That(parsed.ImageIdValid, Is.True); + }); + } + + [Test, Category("Exception")] + public void GivenAV4ImageWithABootSignature_WhenTheKernelIsReplaced_ThenTheEditIsRefusedAndTheImageUntouched() { + var image = Create(("kernel", Bytes(100, 1)), ("boot_signature", Bytes(4096, 2)), ("metadata.ini", Ini("[android_boot]\nheader_version = 4\n"))); + using var ms = new MemoryStream(); + ms.Write(image); + Assert.That(() => Descriptor.Add(ms, [ArchiveInputInfo.InMemory("kernel", [9])]), Throws.TypeOf()); + Assert.That(ms.ToArray(), Is.EqualTo(image)); + } + + [Test, Category("HappyPath")] + public void GivenAV4ImageWithABootSignature_WhenTheSignatureIsRemoved_ThenTheEditIsAllowed() { + var image = Create(("kernel", Bytes(100, 1)), ("boot_signature", Bytes(4096, 2)), ("metadata.ini", Ini("[android_boot]\nheader_version = 4\n"))); + using var ms = new MemoryStream(); + ms.Write(image); + Descriptor.Remove(ms, ["boot_signature"]); + Assert.That(Extract(ms.ToArray()).ContainsKey("boot_signature"), Is.False); + } + + [Test, Category("Exception")] + public void GivenAnImageWithAnAvbFooter_WhenTheRamdiskIsReplaced_ThenTheEditIsRefused() { + var footer = new byte[4096]; + "AVBf"u8.CopyTo(footer.AsSpan(footer.Length - 64)); + var image = Create(("kernel", Bytes(100, 1)), ("trailer.bin", footer)); + using var ms = new MemoryStream(); + ms.Write(image); + Assert.That(() => Descriptor.Add(ms, [ArchiveInputInfo.InMemory("ramdisk", [9])]), Throws.TypeOf()); + Assert.That(ms.ToArray(), Is.EqualTo(image)); + } + + [Test, Category("Exception")] + public void GivenNonZeroPagePadding_WhenEdited_ThenTheEditIsRefusedButTheImageStillReads() { + var image = Create(("kernel", Bytes(100, 1)), ("ramdisk", Bytes(10, 2))); + image[2048 + 100] = 0xEE; // inside the kernel's page padding + using var ms = new MemoryStream(); + ms.Write(image); + Assert.Multiple(() => { + Assert.That(Extract(image)["kernel"], Is.EqualTo(Bytes(100, 1))); + Assert.That(() => Descriptor.Remove(ms, ["ramdisk"]), Throws.TypeOf()); + Assert.That(ms.ToArray(), Is.EqualTo(image)); + }); + } + + [Test, Category("HappyPath")] + public void GivenAForeignImageIdThatIsNotTheSha_WhenEdited_ThenTheIdIsKeptAsItWas() { + var image = Create(("kernel", Bytes(100, 1)), ("ramdisk", Bytes(10, 2))); + image.AsSpan(576, 32).Clear(); + using var ms = new MemoryStream(); + ms.Write(image); + Descriptor.Remove(ms, ["ramdisk"]); + Assert.That(ms.ToArray().AsSpan(576, 32).ToArray(), Is.All.Zero); + } + + [Test, Category("HappyPath")] + public void GivenHeaderBytesPastTheKnownFields_WhenEdited_ThenTheyArePreserved() { + var image = Create(("kernel", Bytes(100, 1)), ("ramdisk", Bytes(10, 2))); + image[2000] = 0x5A; // header page, beyond the v0 fields + using var ms = new MemoryStream(); + ms.Write(image); + Descriptor.Add(ms, [ArchiveInputInfo.InMemory("ramdisk", Bytes(20, 3))]); + Assert.That(ms.ToArray()[2000], Is.EqualTo(0x5A)); + } +} diff --git a/Compression.Tests/AndroidBoot/AndroidVbmetaTests.cs b/Compression.Tests/AndroidBoot/AndroidVbmetaTests.cs new file mode 100644 index 000000000..95320d65d --- /dev/null +++ b/Compression.Tests/AndroidBoot/AndroidVbmetaTests.cs @@ -0,0 +1,235 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Diagnostics; +using System.Text; +using Compression.Registry; +using FileFormat.AndroidVbmeta; +using Win32Exception = System.ComponentModel.Win32Exception; + +namespace Compression.Tests.AndroidBoot; + +/// +/// vbmeta images: two written by avbtool (see Firmware/ReferenceVectors/README.md) are read, +/// verified and — for the unsigned one — rewritten byte for byte; unsigned images we write must +/// match avbtool make_vbmeta_image --algorithm NONE and pass its info_image. +/// +[TestFixture] +public sealed class AndroidVbmetaTests { + + private static readonly AndroidVbmetaFormatDescriptor Descriptor = new(); + + private static Dictionary Extract(byte[] image) { + var dir = Path.Combine(Path.GetTempPath(), "cwb_vbm_" + Guid.NewGuid().ToString("N")[..10]); + try { + using (var ms = new MemoryStream(image)) Descriptor.Extract(ms, dir, null, null); + return Directory.GetFiles(dir, "*", SearchOption.AllDirectories) + .ToDictionary(f => Path.GetRelativePath(dir, f).Replace('\\', '/'), File.ReadAllBytes, StringComparer.Ordinal); + } finally { + try { Directory.Delete(dir, true); } catch { /* best effort */ } + } + } + + private static byte[] Create(IEnumerable<(string Name, byte[] Data)> inputs) { + using var ms = new MemoryStream(); + Descriptor.Create(ms, [.. inputs.Select(i => ArchiveInputInfo.InMemory(i.Name, i.Data))], new FormatCreateOptions()); + return ms.ToArray(); + } + + private static string Ini(byte[] image) => Encoding.UTF8.GetString(Extract(image)["metadata.ini"]); + + // ── avbtool-written vectors ────────────────────────────────────────── + + [Test, Category("HappyPath")] + public void GivenAvbtoolsSignedVbmeta_WhenListed_ThenHashAndRsaSignatureVerify() { + var ini = Ini(Firmware.FirmwareVectors.Load("signed.vbmeta")); + Assert.Multiple(() => { + Assert.That(ini, Does.Contain("algorithm = SHA256_RSA2048")); + Assert.That(ini, Does.Contain("hash_valid = true")); + Assert.That(ini, Does.Contain("signature_valid = true")); + Assert.That(ini, Does.Contain("rollback_index = 7")); + Assert.That(ini, Does.Contain("release_string = avbtool 1.4.0")); + // the values avbtool info_image printed for this image + Assert.That(ini, Does.Contain("public_key_sha1 = cdbb77177f731920bbe0a0f94f84d9038ae0617d")); + Assert.That(ini, Does.Contain("digest = bf2c275accee5b56c848d7708341b4d2f244b2eebdac087f56c2ad65bbe9cd6c")); + Assert.That(ini, Does.Contain("cmdline = androidboot.cwb=1")); + Assert.That(ini, Does.Contain("key = com.cwb.test\nvalue = yes")); + }); + } + + [Test, Category("HappyPath")] + public void GivenAvbtoolsSignedVbmeta_WhenExtracted_ThenEachDescriptorAndTheKeyAreMembers() { + var files = Extract(Firmware.FirmwareVectors.Load("signed.vbmeta")); + Assert.That(files.Keys, Is.SupersetOf(new[] { + "metadata.ini", "descriptors/00_chain_partition_system.bin", "descriptors/01_property_com.cwb.test.bin", + "descriptors/02_kernel_cmdline.bin", "descriptors/03_hash_boot.bin", "public_key.bin", "signature.bin", + })); + Assert.That(files["public_key.bin"].Length, Is.EqualTo(8 + 2 * 256)); + } + + [Test, Category("Exception")] + public void GivenASignedVbmetaWithOneFlippedDescriptorByte_WhenListed_ThenTheHashAndSignatureFail() { + var image = Firmware.FirmwareVectors.Load("signed.vbmeta"); + image[256 + 320 + 100] ^= 0x01; // inside the auxiliary block + var ini = Ini(image); + Assert.That(ini, Does.Contain("hash_valid = false").And.Contain("signature_valid = false")); + } + + [Test, Category("Exception")] + public void GivenASignedVbmeta_WhenEdited_ThenTheEditIsRefusedAndTheImageUntouched() { + var image = Firmware.FirmwareVectors.Load("signed.vbmeta"); + using var ms = new MemoryStream(); + ms.Write(image); + Assert.That(() => Descriptor.Remove(ms, ["descriptors/02_kernel_cmdline.bin"]), Throws.TypeOf()); + Assert.That(ms.ToArray(), Is.EqualTo(image)); + } + + [Test, Category("RoundTrip")] + public void GivenAvbtoolsUnsignedVbmeta_WhenExtractedAndRecreated_ThenTheBytesAreIdentical() { + var image = Firmware.FirmwareVectors.Load("unsigned.vbmeta"); + var rebuilt = Create(Extract(image).Select(kv => (kv.Key, kv.Value))); + Assert.That(rebuilt, Is.EqualTo(image)); + } + + [Test, Category("HappyPath")] + public void GivenAvbtoolsUnsignedVbmeta_WhenADescriptorIsRemoved_ThenTheRestStayAndTheBlockShrinks() { + var image = Firmware.FirmwareVectors.Load("unsigned.vbmeta"); + using var ms = new MemoryStream(); + ms.Write(image); + Descriptor.Remove(ms, ["descriptors/00_property_com.cwb.test.bin"]); + var edited = AvbVbmeta.Parse(ms.ToArray()); + var original = AvbVbmeta.Parse(image); + Assert.Multiple(() => { + Assert.That(edited.Descriptors.Select(d => d.Kind), Is.EqualTo(original.Descriptors.Skip(1).Select(d => d.Kind))); + Assert.That(edited.RollbackIndex, Is.EqualTo(3)); + Assert.That(edited.IsCanonical, Is.True); + }); + } + + // ── malformed / boundary ───────────────────────────────────────────── + + [Test, Category("Exception")] + public void GivenATruncatedVbmeta_WhenListed_ThenItFallsBackWithoutThrowing() { + var image = Firmware.FirmwareVectors.Load("signed.vbmeta")[..1000]; + using var ms = new MemoryStream(image); + Assert.That(Descriptor.List(ms, null).Select(e => e.Name), Is.EqualTo(new[] { "metadata.ini", "FULL.img" })); + } + + [Test, Category("Exception")] + public void GivenADescriptorLengthPastItsBlock_WhenParsed_ThenItIsRejected() { + var image = Firmware.FirmwareVectors.Load("unsigned.vbmeta"); + BinaryPrimitives.WriteUInt64BigEndian(image.AsSpan(256 + 8), 1_000_000); + Assert.That(() => AvbVbmeta.Parse(image), Throws.TypeOf()); + } + + [Test, Category("Boundary")] + public void GivenNoInputs_WhenCreated_ThenTheResultIsABareHeader() { + var image = Create([]); + var parsed = AvbVbmeta.Parse(image); + Assert.Multiple(() => { + Assert.That(image.Length, Is.EqualTo(256)); + Assert.That(parsed.Descriptors, Is.Empty); + Assert.That(parsed.AlgorithmType, Is.Zero); + }); + } + + [Test, Category("Exception")] + public void GivenAPublicKeyAmongTheInputs_WhenCreated_ThenSigningIsRefused() + => Assert.That(() => Create([("public_key.bin", new byte[520])]), Throws.TypeOf()); + + [Test, Category("Exception")] + public void GivenAMalformedDescriptorBlob_WhenCreated_ThenItIsRefused() + => Assert.That(() => Create([("descriptors/00_x.bin", new byte[20])]), Throws.ArgumentException); + + // ── write constraints ──────────────────────────────────────────────── + + [TestCase("descriptors/00_hash.bin")] + [TestCase("descriptors\\03_property.bin")] + [TestCase("public_key_metadata.bin")] + [TestCase("trailer.bin")] + [TestCase("METADATA.INI")] + public void GivenAMemberCreateWrites_WhenOffered_ThenItIsAccepted(string name) { + Assert.That(Descriptor.CanAccept(ArchiveInputInfo.InMemory(name, [1]), out var reason), Is.True); + Assert.That(reason, Is.Null); + } + + [TestCase("PROBE.BIN")] + [TestCase("descriptors/hash.bin")] + [TestCase("00_hash.bin")] + [TestCase("FULL.img")] + public void GivenANameThatIsNoVbmetaMember_WhenOffered_ThenItIsDeclinedWithTheMemberList(string name) { + Assert.That(Descriptor.CanAccept(ArchiveInputInfo.InMemory(name, [1]), out var reason), Is.False); + Assert.That(reason, Does.Contain("descriptors/NN_*.bin")); + Assert.That(() => Create([(name, new byte[] { 1 })]), Throws.ArgumentException, "Create refuses what CanAccept declines"); + } + + [TestCase("public_key.bin")] + [TestCase("signature.bin")] + public void GivenAKeyOrSignature_WhenOffered_ThenItIsDeclinedBecauseSigningNeedsThePrivateKey(string name) { + Assert.That(Descriptor.CanAccept(ArchiveInputInfo.InMemory(name, [1]), out var reason), Is.False); + Assert.That(reason, Does.Contain("private key")); + } + + // ── avbtool oracle ─────────────────────────────────────────────────── + + private static readonly Lazy Avbtool = new(() => { + var configured = Environment.GetEnvironmentVariable("CWB_AVBTOOL"); + var candidates = new List(); + if (!string.IsNullOrWhiteSpace(configured)) { + candidates.Add(["python3", "-c", AndroidBootImageMkbootimgOracleTests.RunScript, configured]); + candidates.Add(["python", "-c", AndroidBootImageMkbootimgOracleTests.RunScript, configured]); + } + candidates.Add(["avbtool"]); + foreach (var c in candidates) + try { + if (Run(c, ["version"]).ExitCode == 0) return c; + } catch (Win32Exception) { + // not there + } + return null; + }); + + private static (int ExitCode, string StdOut, string StdErr) Run(string[] tool, IEnumerable arguments) { + var start = new ProcessStartInfo(tool[0]) { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }; + foreach (var a in tool.Skip(1).Concat(arguments)) start.ArgumentList.Add(a); + using var process = Process.Start(start)!; + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEnd(); + process.WaitForExit(); + return (process.ExitCode, stdout.Result, stderr); + } + + [Test, Category("ArchiveExternal")] + public void GivenDescriptorsAvbtoolWrote_WhenWeWriteAnUnsignedImage_ThenAvbtoolWritesTheSameBytesAndReadsOurs() { + if (Avbtool.Value is not { } avbtool) { + Assert.Ignore("avbtool not found (set CWB_AVBTOOL to avbtool.py from platform/external/avb)."); + return; + } + var dir = Path.Combine(Path.GetTempPath(), "cwb_avbtool_" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(dir); + try { + var reference = Path.Combine(dir, "reference.vbmeta"); + var (code, _, stderr) = Run(avbtool, ["make_vbmeta_image", "--output", reference, "--algorithm", "NONE", + "--prop", "a.b:c", "--prop", "empty:", "--kernel_cmdline", "quiet splash", "--rollback_index", "42", + "--rollback_index_location", "2", "--padding_size", "4096"]); + Assert.That(code, Is.Zero, stderr); + var expected = File.ReadAllBytes(reference); + + var ours = Create(Extract(expected).Select(kv => (kv.Key, kv.Value))); + Assert.That(ours, Is.EqualTo(expected), "rebuilding avbtool's unsigned image from its members changed the bytes"); + + // A descriptor added by us, read back by avbtool. + using var ms = new MemoryStream(); + ms.Write(expected); + var extra = AvbVbmeta.Parse(expected).Descriptors[0].Encoded; + Descriptor.Add(ms, [ArchiveInputInfo.InMemory("descriptors/03_property_copy.bin", extra)]); + var edited = Path.Combine(dir, "edited.vbmeta"); + File.WriteAllBytes(edited, ms.ToArray()); + var (infoCode, info, infoErr) = Run(avbtool, ["info_image", "--image", edited]); + Assert.That(infoCode, Is.Zero, infoErr); + Assert.That(info.Split("Prop: a.b -> 'c'").Length - 1, Is.EqualTo(2), info); + Assert.That(info, Does.Contain("Rollback Index: 42")); + } finally { + try { Directory.Delete(dir, true); } catch { /* best effort */ } + } + } +} diff --git a/Compression.Tests/Compression.Tests.csproj b/Compression.Tests/Compression.Tests.csproj index 45c6258c6..2a73a967c 100644 --- a/Compression.Tests/Compression.Tests.csproj +++ b/Compression.Tests/Compression.Tests.csproj @@ -87,6 +87,10 @@ CscVectors.%(Filename)%(Extension) + + + FirmwareVectors.%(Filename)%(Extension) + FreeArcVectors.%(Filename)%(Extension) diff --git a/Compression.Tests/Firmware/FirmwareVectors.cs b/Compression.Tests/Firmware/FirmwareVectors.cs new file mode 100644 index 000000000..02f09da4a --- /dev/null +++ b/Compression.Tests/Firmware/FirmwareVectors.cs @@ -0,0 +1,15 @@ +#pragma warning disable CS1591 +using System.Reflection; + +namespace Compression.Tests.Firmware; + +/// The reference images under Firmware/ReferenceVectors, embedded as FirmwareVectors.<name>. +internal static class FirmwareVectors { + public static byte[] Load(string name) { + using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream($"FirmwareVectors.{name}") + ?? throw new FileNotFoundException($"embedded firmware vector {name} is missing"); + using var ms = new MemoryStream(); + stream.CopyTo(ms); + return ms.ToArray(); + } +} diff --git a/Compression.Tests/Firmware/ReferenceVectors/README.md b/Compression.Tests/Firmware/ReferenceVectors/README.md new file mode 100644 index 000000000..9714f6509 --- /dev/null +++ b/Compression.Tests/Firmware/ReferenceVectors/README.md @@ -0,0 +1,31 @@ +# Firmware reference vectors + +Small images written by the reference tool of each firmware format, embedded into the test +assembly as `FirmwareVectors.`. + +## vbmeta (Android Verified Boot 2.0) + +Written on 2026-10-19 by `avbtool 1.4.0` (`avbtool.py` from +, revision +`c5066a96caa7bf4150c0a8cc8cc14ab81733fdc7`) under CPython 3.13, with OpenSSL 3.5.5 doing the RSA +signing, using AOSP's public test key `test/data/testkey_rsa2048.pem` from the same checkout: + +```sh +python -c "open('boot.bin','wb').write(b'B'*8192)" +avbtool add_hash_footer --image boot.bin --partition_name boot --partition_size 131072 --algorithm NONE --salt 00112233 +avbtool extract_public_key --key testkey_rsa2048.pem --output rsa2048.avbpubkey +avbtool make_vbmeta_image --output signed.vbmeta --algorithm SHA256_RSA2048 --key testkey_rsa2048.pem \ + --include_descriptors_from_image boot.bin --kernel_cmdline "androidboot.cwb=1" --prop "com.cwb.test:yes" \ + --rollback_index 7 --chain_partition system:1:rsa2048.avbpubkey +avbtool make_vbmeta_image --output unsigned.vbmeta --algorithm NONE \ + --include_descriptors_from_image boot.bin --kernel_cmdline "androidboot.cwb=1" --prop "com.cwb.test:yes" --rollback_index 3 +``` + +`avbtool info_image --image signed.vbmeta` reported public key SHA-1 +`cdbb77177f731920bbe0a0f94f84d9038ae0617d` and boot hash-descriptor digest +`bf2c275accee5b56c848d7708341b4d2f244b2eebdac087f56c2ad65bbe9cd6c`; the tests assert both. + +| File | Bytes | Algorithm | Descriptors | +|------|------:|-----------|-------------| +| `signed.vbmeta` | 2048 | SHA256_RSA2048 | chain partition, property, kernel cmdline, hash | +| `unsigned.vbmeta` | 576 | NONE | property, kernel cmdline, hash | diff --git a/Compression.Tests/Firmware/ReferenceVectors/signed.vbmeta b/Compression.Tests/Firmware/ReferenceVectors/signed.vbmeta new file mode 100644 index 000000000..b84e33e33 Binary files /dev/null and b/Compression.Tests/Firmware/ReferenceVectors/signed.vbmeta differ diff --git a/Compression.Tests/Firmware/ReferenceVectors/unsigned.vbmeta b/Compression.Tests/Firmware/ReferenceVectors/unsigned.vbmeta new file mode 100644 index 000000000..05d48f7e3 Binary files /dev/null and b/Compression.Tests/Firmware/ReferenceVectors/unsigned.vbmeta differ diff --git a/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs b/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs index 587e5a001..6ff05fbc2 100644 --- a/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs +++ b/Compression.Tests/Operations/FilesystemWriteRoundTripTests.cs @@ -55,6 +55,8 @@ public sealed class FilesystemWriteRoundTripTests { + "A ReFS corpus would have to come from Windows.", ["TahoeLafs"] = "R/W is a live capability-backed namespace — no creator can synthesize a Tahoe grid endpoint and root write-capability. " + "TahoeLafsClientTests exercise the gateway contract with an in-process HTTP oracle.", + ["AndroidVbmeta"] = "members are structured AVB descriptors (descriptors/NN_*.bin), so an arbitrary byte blob is not one and the " + + "generic probe cannot be written. AndroidVbmetaTests round-trips real descriptors byte-identical to avbtool.", }; /// diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImage.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImage.cs new file mode 100644 index 000000000..0382a8ab8 --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImage.cs @@ -0,0 +1,534 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Security.Cryptography; + +namespace FileFormat.AndroidBootImage; + +/// One fragment of a vendor_boot ramdisk table (header v4) or the single vendor ramdisk (v3). +public sealed class VendorRamdisk { + public byte[] Data { get; set; } = []; + public uint Type { get; set; } + /// The 32-byte NUL-padded name field, kept raw. + public byte[] Name { get; set; } = new byte[AndroidBootImage.VendorRamdiskNameSize]; + public uint[] BoardId { get; set; } = new uint[AndroidBootImage.BoardIdWords]; +} + +/// +/// An Android boot.img / recovery.img (magic ANDROID!, header v0–v4) or +/// vendor_boot.img (magic VNDRBOOT, header v3–v4), parsed into its header fields and +/// page-aligned components, and written back the way AOSP mkbootimg lays one out. +/// +/// +/// Layout and field offsets follow AOSP system/tools/mkbootimg +/// (include/bootimg/bootimg.h, mkbootimg.py, unpack_bootimg.py; Apache-2.0), +/// converted. All integers are little-endian. +/// Every header byte the parser does not interpret — the rest of the header page(s), the +/// fixed-size string fields past their NUL — is kept and written back unchanged, so an image +/// that is re-laid out without edits comes back byte for byte. Layouts mkbootimg could not +/// have produced (non-zero padding, a relocated recovery DTBO, a sparse ramdisk table) are read +/// but flagged = false, and edits refuse them rather than move bytes. +/// +public sealed class AndroidBootImage { + + public const string BootMagic = "ANDROID!"; + public const string VendorBootMagic = "VNDRBOOT"; + public const int BootNameSize = 16; + public const int BootArgsSize = 512; + public const int BootExtraArgsSize = 1024; + public const int BootV3ArgsSize = BootArgsSize + BootExtraArgsSize; + public const int VendorArgsSize = 2048; + public const int V3PageSize = 4096; + public const int VendorRamdiskNameSize = 32; + public const int BoardIdWords = 16; + public const int VendorRamdiskEntrySize = 108; + public const uint VendorRamdiskTypePlatform = 1; + + /// Header lengths per version, as mkbootimg writes them into the header-size field. + public const uint BootV1HeaderSize = 1648, BootV2HeaderSize = 1660, BootV3HeaderSize = 1580, BootV4HeaderSize = 1584; + public const uint VendorV3HeaderSize = 2112, VendorV4HeaderSize = 2128; + + public bool IsVendor { get; set; } + public uint HeaderVersion { get; set; } + /// Flash page size. Fixed at 4096 for boot v3/v4. + public uint PageSize { get; set; } = 2048; + + public uint KernelAddress { get; set; } + public uint RamdiskAddress { get; set; } + public uint SecondAddress { get; set; } + public uint TagsAddress { get; set; } + public ulong DtbAddress { get; set; } + /// (os_version << 11) | os_patch_level, raw. + public uint OsVersionPatchLevel { get; set; } + public byte[] Board { get; set; } = new byte[BootNameSize]; + /// The command-line field, raw: 512 bytes (boot v0–v2), 1536 (boot v3/v4), 2048 (vendor). + public byte[] Cmdline { get; set; } = new byte[BootArgsSize]; + public byte[] ExtraCmdline { get; set; } = new byte[BootExtraArgsSize]; + public byte[] ImageId { get; set; } = new byte[32]; + /// The header-size field (boot v1+, vendor); 0 where the version has none. + public uint HeaderSize { get; set; } + public uint[] Reserved { get; set; } = new uint[4]; + /// Header-region bytes after the fields this parser knows, up to the first component. + public byte[] HeaderTail { get; set; } = []; + + public byte[] Kernel { get; set; } = []; + public byte[] Ramdisk { get; set; } = []; + public byte[] Second { get; set; } = []; + public byte[] RecoveryDtbo { get; set; } = []; + public byte[] Dtb { get; set; } = []; + public byte[] BootSignature { get; set; } = []; + public byte[] Bootconfig { get; set; } = []; + public List VendorRamdisks { get; set; } = []; + /// Bytes after the last component: partition padding, an AVB footer, anything appended. + public byte[] Trailer { get; set; } = []; + + /// True when the stored image id equals the SHA-1 mkbootimg computes (boot v0–v2). + public bool ImageIdValid { get; private set; } + public bool IsCanonical { get; private set; } = true; + public string? NonCanonicalReason { get; private set; } + + /// True when the trailer ends in an AVB footer (AVBf in the last 64 bytes). + public bool HasAvbFooter => this.Trailer.Length >= 64 + && this.Trailer.AsSpan(this.Trailer.Length - 64, 4).SequenceEqual("AVBf"u8); + + public uint EffectivePageSize => !this.IsVendor && this.HeaderVersion >= 3 ? V3PageSize : this.PageSize; + + public int KnownHeaderLength => this.IsVendor + ? (this.HeaderVersion >= 4 ? (int)VendorV4HeaderSize : (int)VendorV3HeaderSize) + : this.HeaderVersion switch { + 0 => 1632, 1 => (int)BootV1HeaderSize, 2 => (int)BootV2HeaderSize, + 3 => (int)BootV3HeaderSize, _ => (int)BootV4HeaderSize, + }; + + private int HeaderRegionLength { + get { + var page = (int)this.EffectivePageSize; + var declared = this.IsVendor ? Math.Max((int)this.HeaderSize, this.KnownHeaderLength) : page; + return Pages(declared, page) * page; + } + } + + public static bool IsPowerOfTwoPage(uint page) => page is >= 2048 and <= 262144 && (page & (page - 1)) == 0; + + public static int Pages(long size, int page) => checked((int)((size + page - 1) / page)); + + // ── parse ──────────────────────────────────────────────────────────── + + public static AndroidBootImage Parse(ReadOnlySpan data) { + if (data.Length < 8) throw new InvalidDataException("Android boot image: shorter than the magic."); + var img = new AndroidBootImage(); + if (data[..8].SequenceEqual("ANDROID!"u8)) img.ParseBoot(data); + else if (data[..8].SequenceEqual("VNDRBOOT"u8)) img.ParseVendor(data); + else throw new InvalidDataException("Android boot image: magic is neither ANDROID! nor VNDRBOOT."); + return img; + } + + private static uint U32(ReadOnlySpan d, int o) => BinaryPrimitives.ReadUInt32LittleEndian(d[o..]); + private static ulong U64(ReadOnlySpan d, int o) => BinaryPrimitives.ReadUInt64LittleEndian(d[o..]); + + private static void Need(ReadOnlySpan d, long end, string what) { + if (end > d.Length) + throw new InvalidDataException($"Android boot image: {what} runs to byte {end}, past the end of the {d.Length}-byte file."); + } + + private void ParseBoot(ReadOnlySpan d) { + Need(d, 44, "header"); + this.HeaderVersion = U32(d, 40); + if (this.HeaderVersion > 4) throw new InvalidDataException($"Android boot image: header version {this.HeaderVersion} is unknown."); + Need(d, this.KnownHeaderLength, "header"); + + uint kernelSize, ramdiskSize, secondSize = 0, recoveryDtboSize = 0, dtbSize = 0, signatureSize = 0; + ulong recoveryDtboOffset = 0; + if (this.HeaderVersion < 3) { + kernelSize = U32(d, 8); + this.KernelAddress = U32(d, 12); + ramdiskSize = U32(d, 16); + this.RamdiskAddress = U32(d, 20); + secondSize = U32(d, 24); + this.SecondAddress = U32(d, 28); + this.TagsAddress = U32(d, 32); + this.PageSize = U32(d, 36); + if (!IsPowerOfTwoPage(this.PageSize)) + throw new InvalidDataException($"Android boot image: page size {this.PageSize} is not a power of two in 2048..262144."); + this.OsVersionPatchLevel = U32(d, 44); + this.Board = d.Slice(48, BootNameSize).ToArray(); + this.Cmdline = d.Slice(64, BootArgsSize).ToArray(); + this.ImageId = d.Slice(576, 32).ToArray(); + this.ExtraCmdline = d.Slice(608, BootExtraArgsSize).ToArray(); + if (this.HeaderVersion >= 1) { + recoveryDtboSize = U32(d, 1632); + recoveryDtboOffset = U64(d, 1636); + this.HeaderSize = U32(d, 1644); + } + if (this.HeaderVersion == 2) { + dtbSize = U32(d, 1648); + this.DtbAddress = U64(d, 1652); + } + } else { + this.PageSize = V3PageSize; + kernelSize = U32(d, 8); + ramdiskSize = U32(d, 12); + this.OsVersionPatchLevel = U32(d, 16); + this.HeaderSize = U32(d, 20); + for (var i = 0; i < 4; ++i) this.Reserved[i] = U32(d, 24 + 4 * i); + this.Cmdline = d.Slice(44, BootV3ArgsSize).ToArray(); + if (this.HeaderVersion >= 4) signatureSize = U32(d, 1580); + } + + var page = (int)this.EffectivePageSize; + var region = this.HeaderRegionLength; + Need(d, region, "header page"); + this.HeaderTail = d[this.KnownHeaderLength..region].ToArray(); + + long offset = region; + this.Kernel = this.Take(d, ref offset, kernelSize, page, "kernel"); + this.Ramdisk = this.Take(d, ref offset, ramdiskSize, page, "ramdisk"); + if (this.HeaderVersion >= 4 && signatureSize > 0) + this.BootSignature = this.Take(d, ref offset, signatureSize, page, "boot signature"); + this.Second = this.Take(d, ref offset, secondSize, page, "second-stage bootloader"); + if (recoveryDtboSize > 0) { + if ((ulong)offset != recoveryDtboOffset) { + // unpack_bootimg honours the stored offset; so do we, but such an image is not ours to re-lay out. + this.MarkNonCanonical($"recovery DTBO sits at {recoveryDtboOffset}, not at {offset} where mkbootimg puts it"); + Need(d, (long)recoveryDtboOffset + recoveryDtboSize, "recovery DTBO"); + this.RecoveryDtbo = d.Slice(checked((int)recoveryDtboOffset), (int)recoveryDtboSize).ToArray(); + offset = Math.Max(offset, (long)recoveryDtboOffset + Pages(recoveryDtboSize, page) * (long)page); + } else + this.RecoveryDtbo = this.Take(d, ref offset, recoveryDtboSize, page, "recovery DTBO"); + } + this.Dtb = this.Take(d, ref offset, dtbSize, page, "DTB"); + this.TakeTrailer(d, offset); + + if (this.HeaderVersion < 3) { + this.ImageIdValid = this.ComputeImageId().AsSpan().SequenceEqual(this.ImageId); + // mkbootimg leaves a zero offset when there is no recovery DTBO; anything else would be lost on rebuild. + if (this.HeaderVersion >= 1 && recoveryDtboSize == 0 && recoveryDtboOffset != 0) + this.MarkNonCanonical("an empty recovery DTBO carries a non-zero offset"); + } + } + + private void ParseVendor(ReadOnlySpan d) { + this.IsVendor = true; + Need(d, 12, "header"); + this.HeaderVersion = U32(d, 8); + if (this.HeaderVersion is < 3 or > 4) + throw new InvalidDataException($"Android vendor_boot image: header version {this.HeaderVersion} is unknown."); + Need(d, this.KnownHeaderLength, "header"); + this.PageSize = U32(d, 12); + if (!IsPowerOfTwoPage(this.PageSize)) + throw new InvalidDataException($"Android vendor_boot image: page size {this.PageSize} is not a power of two in 2048..262144."); + this.KernelAddress = U32(d, 16); + this.RamdiskAddress = U32(d, 20); + var ramdiskTotal = U32(d, 24); + this.Cmdline = d.Slice(28, VendorArgsSize).ToArray(); + this.TagsAddress = U32(d, 2076); + this.Board = d.Slice(2080, BootNameSize).ToArray(); + this.HeaderSize = U32(d, 2096); + var dtbSize = U32(d, 2100); + this.DtbAddress = U64(d, 2104); + uint tableSize = 0, entryCount = 0, entrySize = 0, bootconfigSize = 0; + if (this.HeaderVersion >= 4) { + tableSize = U32(d, 2112); + entryCount = U32(d, 2116); + entrySize = U32(d, 2120); + bootconfigSize = U32(d, 2124); + } + if (this.HeaderSize > 1 << 20) throw new InvalidDataException($"Android vendor_boot image: header size {this.HeaderSize} is implausible."); + + var page = (int)this.PageSize; + var region = this.HeaderRegionLength; + Need(d, region, "header pages"); + this.HeaderTail = d[this.KnownHeaderLength..region].ToArray(); + + long offset = region; + var ramdiskBase = offset; + Need(d, offset + ramdiskTotal, "vendor ramdisk"); + var ramdisks = d.Slice((int)offset, (int)ramdiskTotal); + offset += Pages(ramdiskTotal, page) * (long)page; + this.CheckPadding(d, ramdiskBase + ramdiskTotal, offset, "vendor ramdisk"); + this.Dtb = this.Take(d, ref offset, dtbSize, page, "DTB"); + + if (this.HeaderVersion < 4) { + this.VendorRamdisks.Add(new VendorRamdisk { Data = ramdisks.ToArray(), Type = VendorRamdiskTypePlatform }); + } else { + if (entrySize != VendorRamdiskEntrySize && entryCount > 0) + this.MarkNonCanonical($"ramdisk table entries are {entrySize} bytes, not {VendorRamdiskEntrySize}"); + if ((long)entryCount * entrySize > tableSize || entrySize < VendorRamdiskEntrySize && entryCount > 0) + throw new InvalidDataException($"Android vendor_boot image: {entryCount} ramdisk table entries of {entrySize} bytes do not fit the {tableSize}-byte table."); + if (tableSize != entryCount * VendorRamdiskEntrySize) + this.MarkNonCanonical($"ramdisk table is {tableSize} bytes for {entryCount} entries"); + var tableStart = offset; + Need(d, tableStart + tableSize, "ramdisk table"); + long expected = 0; + for (var i = 0; i < entryCount; ++i) { + var e = d.Slice((int)(tableStart + i * (long)entrySize), VendorRamdiskEntrySize); + var size = U32(e, 0); + var rel = U32(e, 4); + if ((ulong)rel + size > ramdiskTotal) + throw new InvalidDataException($"Android vendor_boot image: ramdisk table entry {i} runs past the {ramdiskTotal}-byte ramdisk section."); + if (rel != expected) this.MarkNonCanonical($"ramdisk table entry {i} starts at {rel}, not at {expected}"); + expected = rel + (long)size; + var boardId = new uint[BoardIdWords]; + for (var w = 0; w < BoardIdWords; ++w) boardId[w] = U32(e, 44 + 4 * w); + this.VendorRamdisks.Add(new VendorRamdisk { + Data = ramdisks.Slice((int)rel, (int)size).ToArray(), + Type = U32(e, 8), + Name = e.Slice(12, VendorRamdiskNameSize).ToArray(), + BoardId = boardId, + }); + } + if (expected != ramdiskTotal) this.MarkNonCanonical("the ramdisk table does not cover the whole ramdisk section"); + offset = tableStart + Pages(tableSize, page) * (long)page; + this.CheckPadding(d, tableStart + tableSize, offset, "ramdisk table"); + this.Bootconfig = this.Take(d, ref offset, bootconfigSize, page, "bootconfig"); + } + this.TakeTrailer(d, offset); + } + + private byte[] Take(ReadOnlySpan d, ref long offset, uint size, int page, string what) { + if (size == 0) return []; + Need(d, offset + size, what); + var result = d.Slice((int)offset, (int)size).ToArray(); + var end = offset + Pages(size, page) * (long)page; + this.CheckPadding(d, offset + size, end, what); + offset = end; + return result; + } + + private void CheckPadding(ReadOnlySpan d, long from, long to, string what) { + var end = Math.Min(to, d.Length); + if (from < end && d[(int)from..(int)end].ContainsAnyExcept((byte)0)) + this.MarkNonCanonical($"the page padding after the {what} is not zero"); + } + + private void TakeTrailer(ReadOnlySpan d, long offset) { + // A final component whose page padding was cut off is still complete: the image ends at its data. + this.Trailer = offset < d.Length ? d[(int)offset..].ToArray() : []; + } + + private void MarkNonCanonical(string reason) { + this.IsCanonical = false; + this.NonCanonicalReason ??= reason; + } + + // ── image id ───────────────────────────────────────────────────────── + + /// + /// The boot v0–v2 image id exactly as mkbootimg computes it: SHA-1 over each component + /// followed by its 32-bit length (kernel, ramdisk, second, then recovery DTBO from v1 and DTB + /// from v2), zero-extended to 32 bytes. + /// + public byte[] ComputeImageId() { + using var sha = IncrementalHash.CreateHash(HashAlgorithmName.SHA1); + var size = new byte[4]; + void Add(byte[] part) { + sha.AppendData(part); + BinaryPrimitives.WriteUInt32LittleEndian(size, (uint)part.Length); + sha.AppendData(size); + } + Add(this.Kernel); + Add(this.Ramdisk); + Add(this.Second); + if (this.HeaderVersion > 0) Add(this.RecoveryDtbo); + if (this.HeaderVersion > 1) Add(this.Dtb); + var id = new byte[32]; + sha.GetHashAndReset().CopyTo(id, 0); + return id; + } + + // ── write ──────────────────────────────────────────────────────────── + + /// + /// Writes the image in mkbootimg order: header page(s), then each component padded to the + /// page size. A boot v0–v2 image id is recomputed when is set + /// (a fresh image, or one whose stored id was valid), else kept as it was. + /// + public void Write(Stream output, bool recomputeImageId) { + ArgumentNullException.ThrowIfNull(output); + this.Validate(); + var page = (int)this.EffectivePageSize; + var region = this.HeaderRegionLength; + var header = new byte[region]; + if (recomputeImageId && !this.IsVendor && this.HeaderVersion < 3) this.ImageId = this.ComputeImageId(); + if (this.IsVendor) this.WriteVendorHeader(header); + else this.WriteBootHeader(header, region); + this.HeaderTail.AsSpan(0, Math.Min(this.HeaderTail.Length, region - this.KnownHeaderLength)) + .CopyTo(header.AsSpan(this.KnownHeaderLength)); + output.Write(header); + + if (this.IsVendor) { + foreach (var r in this.VendorRamdisks) output.Write(r.Data); + Pad(output, this.VendorRamdisks.Sum(r => (long)r.Data.Length), page); + WritePadded(output, this.Dtb, page); + if (this.HeaderVersion >= 4) { + var table = new byte[this.VendorRamdisks.Count * VendorRamdiskEntrySize]; + uint rel = 0; + for (var i = 0; i < this.VendorRamdisks.Count; ++i) { + var r = this.VendorRamdisks[i]; + var e = table.AsSpan(i * VendorRamdiskEntrySize, VendorRamdiskEntrySize); + BinaryPrimitives.WriteUInt32LittleEndian(e, (uint)r.Data.Length); + BinaryPrimitives.WriteUInt32LittleEndian(e[4..], rel); + BinaryPrimitives.WriteUInt32LittleEndian(e[8..], r.Type); + r.Name.AsSpan(0, Math.Min(r.Name.Length, VendorRamdiskNameSize)).CopyTo(e[12..]); + for (var w = 0; w < BoardIdWords; ++w) BinaryPrimitives.WriteUInt32LittleEndian(e[(44 + 4 * w)..], r.BoardId[w]); + rel += (uint)r.Data.Length; + } + output.Write(table); + Pad(output, table.Length, page); + WritePadded(output, this.Bootconfig, page); + } + } else { + WritePadded(output, this.Kernel, page); + WritePadded(output, this.Ramdisk, page); + if (this.HeaderVersion >= 4) WritePadded(output, this.BootSignature, page); + WritePadded(output, this.Second, page); + if (this.HeaderVersion is 1 or 2) WritePadded(output, this.RecoveryDtbo, page); + if (this.HeaderVersion == 2) WritePadded(output, this.Dtb, page); + } + output.Write(this.Trailer); + } + + private void Validate() { + if (this.IsVendor ? this.HeaderVersion is < 3 or > 4 : this.HeaderVersion > 4) + throw new InvalidOperationException($"Android {(this.IsVendor ? "vendor_boot" : "boot")} image: header version {this.HeaderVersion} is not supported."); + if (!IsPowerOfTwoPage(this.EffectivePageSize)) + throw new InvalidOperationException($"Android boot image: page size {this.EffectivePageSize} is not a power of two in 2048..262144."); + if (!this.IsVendor) { + if (this.HeaderVersion >= 3 && this.Second.Length > 0) + throw new InvalidOperationException("Android boot image: header v3/v4 has no second-stage bootloader."); + if (this.HeaderVersion is not (1 or 2) && this.RecoveryDtbo.Length > 0) + throw new InvalidOperationException($"Android boot image: header v{this.HeaderVersion} has no recovery DTBO."); + if (this.HeaderVersion != 2 && this.Dtb.Length > 0) + throw new InvalidOperationException($"Android boot image: header v{this.HeaderVersion} has no DTB (only v2, or vendor_boot)."); + if (this.HeaderVersion < 4 && this.BootSignature.Length > 0) + throw new InvalidOperationException("Android boot image: only header v4 carries a boot signature."); + if (this.Bootconfig.Length > 0 || this.VendorRamdisks.Count > 0) + throw new InvalidOperationException("Android boot image: vendor ramdisks and bootconfig belong in a vendor_boot image."); + } else { + if (this.Kernel.Length + this.Ramdisk.Length + this.Second.Length + this.RecoveryDtbo.Length + this.BootSignature.Length > 0) + throw new InvalidOperationException("Android vendor_boot image: kernel, ramdisk, second, recovery DTBO and boot signature belong in a boot image."); + if (this.HeaderVersion < 4 && (this.VendorRamdisks.Count > 1 || this.Bootconfig.Length > 0)) + throw new InvalidOperationException("Android vendor_boot image: header v3 holds one vendor ramdisk and no bootconfig."); + } + } + + private void WriteBootHeader(Span h, int region) { + "ANDROID!"u8.CopyTo(h); + if (this.HeaderVersion < 3) { + W32(h, 8, (uint)this.Kernel.Length); + W32(h, 12, this.KernelAddress); + W32(h, 16, (uint)this.Ramdisk.Length); + W32(h, 20, this.RamdiskAddress); + W32(h, 24, (uint)this.Second.Length); + W32(h, 28, this.SecondAddress); + W32(h, 32, this.TagsAddress); + W32(h, 36, this.PageSize); + W32(h, 40, this.HeaderVersion); + W32(h, 44, this.OsVersionPatchLevel); + CopyField(this.Board, h.Slice(48, BootNameSize)); + CopyField(this.Cmdline, h.Slice(64, BootArgsSize)); + CopyField(this.ImageId, h.Slice(576, 32)); + CopyField(this.ExtraCmdline, h.Slice(608, BootExtraArgsSize)); + if (this.HeaderVersion >= 1) { + var page = (long)this.PageSize; + W32(h, 1632, (uint)this.RecoveryDtbo.Length); + var dtboOffset = this.RecoveryDtbo.Length == 0 ? 0UL : (ulong)(region + + Pages(this.Kernel.Length, (int)page) * page + Pages(this.Ramdisk.Length, (int)page) * page + + Pages(this.Second.Length, (int)page) * page); + BinaryPrimitives.WriteUInt64LittleEndian(h[1636..], dtboOffset); + W32(h, 1644, this.HeaderSize); + } + if (this.HeaderVersion == 2) { + W32(h, 1648, (uint)this.Dtb.Length); + BinaryPrimitives.WriteUInt64LittleEndian(h[1652..], this.DtbAddress); + } + } else { + W32(h, 8, (uint)this.Kernel.Length); + W32(h, 12, (uint)this.Ramdisk.Length); + W32(h, 16, this.OsVersionPatchLevel); + W32(h, 20, this.HeaderSize); + for (var i = 0; i < 4; ++i) W32(h, 24 + 4 * i, this.Reserved[i]); + W32(h, 40, this.HeaderVersion); + CopyField(this.Cmdline, h.Slice(44, BootV3ArgsSize)); + if (this.HeaderVersion >= 4) W32(h, 1580, (uint)this.BootSignature.Length); + } + } + + private void WriteVendorHeader(Span h) { + "VNDRBOOT"u8.CopyTo(h); + W32(h, 8, this.HeaderVersion); + W32(h, 12, this.PageSize); + W32(h, 16, this.KernelAddress); + W32(h, 20, this.RamdiskAddress); + W32(h, 24, checked((uint)this.VendorRamdisks.Sum(r => (long)r.Data.Length))); + CopyField(this.Cmdline, h.Slice(28, VendorArgsSize)); + W32(h, 2076, this.TagsAddress); + CopyField(this.Board, h.Slice(2080, BootNameSize)); + W32(h, 2096, this.HeaderSize); + W32(h, 2100, (uint)this.Dtb.Length); + BinaryPrimitives.WriteUInt64LittleEndian(h[2104..], this.DtbAddress); + if (this.HeaderVersion >= 4) { + W32(h, 2112, (uint)(this.VendorRamdisks.Count * VendorRamdiskEntrySize)); + W32(h, 2116, (uint)this.VendorRamdisks.Count); + W32(h, 2120, VendorRamdiskEntrySize); + W32(h, 2124, (uint)this.Bootconfig.Length); + } + } + + private static void W32(Span h, int offset, uint value) => BinaryPrimitives.WriteUInt32LittleEndian(h[offset..], value); + + private static void CopyField(byte[] value, Span field) { + field.Clear(); + value.AsSpan(0, Math.Min(value.Length, field.Length)).CopyTo(field); + } + + private static void WritePadded(Stream output, byte[] data, int page) { + output.Write(data); + Pad(output, data.Length, page); + } + + private static void Pad(Stream output, long written, int page) { + var pad = (int)((page - written % page) % page); + if (pad > 0) output.Write(new byte[pad]); + } + + // ── os version / patch level ───────────────────────────────────────── + + public static string FormatOsVersion(uint packed) { + var v = packed >> 11; + return $"{v >> 14}.{(v >> 7) & 0x7F}.{v & 0x7F}"; + } + + public static string FormatOsPatchLevel(uint packed) { + var p = packed & 0x7FF; + return $"{2000 + (p >> 4):D4}-{p & 0xF:D2}"; + } + + /// Parses a[.b[.c]], 7 bits per field, as mkbootimg --os_version does. + public static bool TryParseOsVersion(string text, out uint version) { + version = 0; + var parts = text.Trim().Split('.'); + if (parts.Length is < 1 or > 3) return false; + uint packed = 0; + for (var i = 0; i < 3; ++i) { + uint field = 0; + if (i < parts.Length && (!uint.TryParse(parts[i], System.Globalization.NumberStyles.None, System.Globalization.CultureInfo.InvariantCulture, out field) || field > 127)) + return false; + packed = (packed << 7) | field; + } + version = packed; + return true; + } + + /// Parses YYYY-MM[-DD] as mkbootimg --os_patch_level does (7 bits of year past 2000, 4 of month). + public static bool TryParseOsPatchLevel(string text, out uint level) { + level = 0; + var parts = text.Trim().Split('-'); + if (parts.Length is < 2 or > 3 + || !uint.TryParse(parts[0], System.Globalization.NumberStyles.None, System.Globalization.CultureInfo.InvariantCulture, out var year) + || !uint.TryParse(parts[1], System.Globalization.NumberStyles.None, System.Globalization.CultureInfo.InvariantCulture, out var month) + || year is < 2000 or > 2127 || month > 15) + return false; + level = ((year - 2000) << 4) | month; + return true; + } +} diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImageFormatDescriptor.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImageFormatDescriptor.cs new file mode 100644 index 000000000..2b81daab5 --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImageFormatDescriptor.cs @@ -0,0 +1,299 @@ +#pragma warning disable CS1591 +using System.Globalization; +using Compression.Registry; +using FileFormat.FirmwareCommon; + +namespace FileFormat.AndroidBootImage; + +/// +/// Android boot image — boot.img / recovery.img (magic ANDROID!, header +/// v0–v4) and vendor_boot.img (magic VNDRBOOT, header v3–v4) — as a pseudo-archive +/// whose members are the components unpack_bootimg extracts, under the names it gives them: +/// kernel, ramdisk, second, recovery_dtbo, dtb, +/// boot_signature, vendor_ramdisk / vendor_ramdiskNN, bootconfig. +/// metadata.ini renders the header; bytes after the last component (partition padding, an +/// AVB footer) surface as trailer.bin. +/// +/// +/// Create and edit write exactly what AOSP mkbootimg writes for the same inputs and +/// header fields, image id included; the oracle tests compare the bytes. An edit refuses an image +/// whose content is signed — a v4 boot signature or an AVB footer would go stale — unless the edit +/// is removing that signature, and it refuses a layout mkbootimg could not have produced. +/// Reference: https://source.android.com/docs/core/architecture/bootloader/boot-image-header; +/// AOSP system/tools/mkbootimg (Apache-2.0), converted. +/// +public sealed class AndroidBootImageFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, + IArchiveCreatable, IArchiveModifiable, ISyntheticEntryNames { + + public const string Kernel = "kernel", Ramdisk = "ramdisk", Second = "second", RecoveryDtbo = "recovery_dtbo", + Dtb = "dtb", BootSignature = "boot_signature", VendorRamdisk = "vendor_ramdisk", Bootconfig = "bootconfig", + Trailer = "trailer.bin", FullImage = "FULL.img"; + + private static readonly IReadOnlySet Synthetic = new HashSet(StringComparer.OrdinalIgnoreCase) { + FirmwareArchive.MetadataName, FullImage, + }; + + public IReadOnlySet SyntheticEntryNames => Synthetic; + public string Id => "AndroidBootImage"; + public string DisplayName => "Android boot image"; + public FormatCategory Category => FormatCategory.Archive; + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | + FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; + public string DefaultExtension => ".bootimg"; + /// .img belongs to every disk-image format; detection is by magic. + public IReadOnlyList Extensions => [".bootimg"]; + public IReadOnlyList CompoundExtensions => []; + public IReadOnlyList MagicSignatures => [ + new("ANDROID!"u8.ToArray(), Confidence: 0.95), + new("VNDRBOOT"u8.ToArray(), Confidence: 0.95), + ]; + public IReadOnlyList Methods => [new("stored", "Stored")]; + public string? TarCompressionFormatId => null; + public AlgorithmFamily Family => AlgorithmFamily.Archive; + public string Description => + "Android boot / recovery (header v0-v4) and vendor_boot (v3-v4) image: kernel, ramdisks, second stage, DTB, " + + "recovery DTBO, bootconfig and boot signature, laid out and identified exactly as mkbootimg does."; + + // ── read ───────────────────────────────────────────────────────────── + + public List List(Stream stream, string? password) => FirmwareArchive.List(Entries(FirmwareArchive.ReadAll(stream))); + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) + => FirmwareArchive.Extract(Entries(FirmwareArchive.ReadAll(stream)), outputDir, files); + + List IArchiveFormatOperations.ListSpan(ReadOnlySpan archive, string? password) + => FirmwareArchive.List(Entries(archive.ToArray())); + + void IArchiveFormatOperations.ExtractSpan(ReadOnlySpan archive, string outputDir, string? password, string[]? files) + => FirmwareArchive.Extract(Entries(archive.ToArray()), outputDir, files); + + /// The members of ; an image that does not parse lists as itself plus the reason. + public static List Entries(byte[] data) { + AndroidBootImage img; + try { + img = AndroidBootImage.Parse(data); + } catch (Exception e) when (e is InvalidDataException or ArgumentException or OverflowException) { + return [new(FirmwareArchive.MetadataName, AndroidBootImageMetadata.RenderPartial(e.Message)), new(FullImage, data)]; + } + var entries = new List { new(FirmwareArchive.MetadataName, AndroidBootImageMetadata.Render(img)) }; + void Add(string name, byte[] part) { + if (part.Length > 0) entries.Add(new(name, part)); + } + if (img.IsVendor) { + if (img.HeaderVersion < 4) Add(VendorRamdisk, img.VendorRamdisks.Count > 0 ? img.VendorRamdisks[0].Data : []); + else + for (var i = 0; i < img.VendorRamdisks.Count; ++i) + entries.Add(new(AndroidBootImageMetadata.RamdiskEntryName(i), img.VendorRamdisks[i].Data)); + Add(Dtb, img.Dtb); + Add(Bootconfig, img.Bootconfig); + } else { + Add(Kernel, img.Kernel); + Add(Ramdisk, img.Ramdisk); + Add(Second, img.Second); + Add(RecoveryDtbo, img.RecoveryDtbo); + Add(Dtb, img.Dtb); + Add(BootSignature, img.BootSignature); + } + Add(Trailer, img.Trailer); + return entries; + } + + // ── create ─────────────────────────────────────────────────────────── + + /// + /// Builds an image from component files named as unpack_bootimg names them, with the + /// header fields taken from a metadata.ini among the inputs (the rendering this + /// descriptor lists) and mkbootimg's defaults for every field it does not set. A single + /// input under any other name is the payload a boot image exists to carry and becomes the + /// kernel, as mkbootimg --kernel would take it; more than one such input is refused. + /// + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + var files = inputs.Where(i => !i.IsDirectory && !string.IsNullOrEmpty(i.ArchiveName)) + .Select(i => (Name: Leaf(i.ArchiveName), Input: i)).ToList(); + var unnamed = files.FindAll(f => !IsComponentName(f.Name) && !IsReservedName(f.Name)); + if (unnamed.Count == 1 && !files.Exists(f => f.Name.Equals(Kernel, StringComparison.OrdinalIgnoreCase))) + files[files.IndexOf(unnamed[0])] = (Kernel, unnamed[0].Input); + foreach (var (name, _) in files) + if (!IsComponentName(name) && !name.Equals(FirmwareArchive.MetadataName, StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException($"Android boot image: '{name}' is not a boot image component " + + "(kernel, ramdisk, second, recovery_dtbo, dtb, boot_signature, vendor_ramdisk[NN], bootconfig, trailer.bin, metadata.ini)."); + + var metadata = files.FirstOrDefault(f => f.Name.Equals(FirmwareArchive.MetadataName, StringComparison.OrdinalIgnoreCase)).Input; + var ini = metadata is null ? null : FirmwareArchive.ParseIni(metadata.ReadContent()); + var vendorHint = files.Any(f => f.Name.StartsWith(VendorRamdisk, StringComparison.OrdinalIgnoreCase) || f.Name.Equals(Bootconfig, StringComparison.OrdinalIgnoreCase)); + var img = AndroidBootImageMetadata.NewImage(ini, vendorHint); + + var parts = files.Where(f => IsComponentName(f.Name)).Select(f => (f.Name, Data: f.Input.ReadContent())).ToList(); + ApplyComponents(img, parts, creating: true); + // mkbootimg writes a zero load address for an absent ramdisk or second stage (boot v0-v2). + if (!img.IsVendor && img.HeaderVersion < 3) { + if (img.Ramdisk.Length == 0) img.RamdiskAddress = 0; + if (img.Second.Length == 0) img.SecondAddress = 0; + } + if (ini is not null) AndroidBootImageMetadata.Apply(img, ini); + Write(output, img, recomputeImageId: true); + } + + // ── modify ─────────────────────────────────────────────────────────── + + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + var (img, original) = OpenForEdit(archive); + var files = inputs.Where(i => !i.IsDirectory && !string.IsNullOrEmpty(i.ArchiveName)) + .Select(i => (Name: Leaf(i.ArchiveName), Input: i)).ToList(); + var parts = new List<(string Name, byte[] Data)>(); + Dictionary? ini = null; + foreach (var (name, input) in files) { + if (name.Equals(FirmwareArchive.MetadataName, StringComparison.OrdinalIgnoreCase)) ini = FirmwareArchive.ParseIni(input.ReadContent()); + else if (name.Equals(FullImage, StringComparison.OrdinalIgnoreCase)) throw new NotSupportedException($"Android boot image: {FullImage} is a view of the whole image, not a member."); + else if (IsComponentName(name)) parts.Add((name, input.ReadContent())); + else throw new ArgumentException($"Android boot image: '{name}' is not a boot image component."); + } + if (parts.Count == 0 && ini is null) return; + RefuseSignedEdit(img, touchesContent: true); + ApplyComponents(img, parts, creating: false); + if (ini is not null) AndroidBootImageMetadata.Apply(img, ini); + Commit(archive, img, original); + } + + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(archive); + var names = (entryNames ?? []).Select(Leaf).Where(n => !Synthetic.Contains(n)).ToList(); + if (names.Count == 0) return; + var (img, original) = OpenForEdit(archive); + var dropsOnlySignatures = names.All(n => n.Equals(BootSignature, StringComparison.OrdinalIgnoreCase) || n.Equals(Trailer, StringComparison.OrdinalIgnoreCase)); + RefuseSignedEdit(img, touchesContent: !dropsOnlySignatures); + + // Remove vendor ramdisk fragments from the highest index down so earlier indices keep their meaning. + var fragments = new List(); + foreach (var name in names) { + if (TryRamdiskIndex(name, out var index)) { + if (index < img.VendorRamdisks.Count) fragments.Add(index); + continue; + } + switch (name.ToLowerInvariant()) { + case Kernel: img.Kernel = []; break; + case Ramdisk: img.Ramdisk = []; break; + case Second: img.Second = []; break; + case RecoveryDtbo: img.RecoveryDtbo = []; break; + case Dtb: img.Dtb = []; break; + case BootSignature: img.BootSignature = []; break; + case Bootconfig: img.Bootconfig = []; break; + case Trailer: img.Trailer = []; break; + case VendorRamdisk: + if (img.IsVendor && img.HeaderVersion < 4) img.VendorRamdisks.Clear(); + break; + } + } + foreach (var index in fragments.Distinct().OrderDescending()) img.VendorRamdisks.RemoveAt(index); + Commit(archive, img, original); + } + + /// Empties every component and the trailer, keeping the header fields: a valid image with nothing in it. + public void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + var (img, original) = OpenForEdit(archive); + img.Kernel = img.Ramdisk = img.Second = img.RecoveryDtbo = img.Dtb = img.BootSignature = img.Bootconfig = img.Trailer = []; + img.VendorRamdisks.Clear(); + Commit(archive, img, original); + } + + private static (AndroidBootImage Image, AndroidBootImage Original) OpenForEdit(Stream archive) { + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("Android boot image editing requires a readable, writable, seekable stream.", nameof(archive)); + var data = FirmwareArchive.ReadAll(archive); + var img = AndroidBootImage.Parse(data); + if (!img.IsCanonical) + throw new NotSupportedException($"Android boot image: {img.NonCanonicalReason}; this layout is not one mkbootimg writes, so it is not re-laid out."); + return (img, AndroidBootImage.Parse(data)); + } + + private static void RefuseSignedEdit(AndroidBootImage img, bool touchesContent) { + if (!touchesContent) return; + if (img.BootSignature.Length > 0) + throw new NotSupportedException("Android boot image: the v4 boot signature covers this image and would go stale; remove boot_signature first."); + if (img.HasAvbFooter) + throw new NotSupportedException("Android boot image: an AVB footer covers this image and would go stale; remove trailer.bin first."); + } + + private static void Commit(Stream archive, AndroidBootImage img, AndroidBootImage original) + => WholeImageRebuildCommitter.Replace(archive, + candidate => Write(candidate, img, recomputeImageId: original.ImageIdValid), + candidate => _ = AndroidBootImage.Parse(FirmwareArchive.ReadAll(candidate))); + + private static void Write(Stream output, AndroidBootImage img, bool recomputeImageId) { + try { + img.Write(output, recomputeImageId); + } catch (InvalidOperationException e) { + throw new ArgumentException(e.Message, e); + } + } + + // ── components ─────────────────────────────────────────────────────── + + private static void ApplyComponents(AndroidBootImage img, List<(string Name, byte[] Data)> parts, bool creating) { + foreach (var (name, data) in parts.Where(p => !TryRamdiskIndex(p.Name, out _))) { + switch (name.ToLowerInvariant()) { + case Kernel: img.Kernel = data; break; + case Ramdisk: img.Ramdisk = data; break; + case Second: img.Second = data; break; + case RecoveryDtbo: img.RecoveryDtbo = data; break; + case Dtb: img.Dtb = data; break; + case BootSignature: img.BootSignature = data; break; + case Bootconfig: img.Bootconfig = data; break; + case Trailer: img.Trailer = data; break; + case VendorRamdisk: + if (img.IsVendor && img.HeaderVersion >= 4) + throw new ArgumentException("Android vendor_boot v4: name ramdisk fragments vendor_ramdisk00, vendor_ramdisk01, ..."); + if (img.VendorRamdisks.Count == 0) img.VendorRamdisks.Add(new VendorRamdisk { Type = AndroidBootImage.VendorRamdiskTypePlatform }); + img.VendorRamdisks[0].Data = data; + break; + } + } + + var fragments = parts.Where(p => TryRamdiskIndex(p.Name, out _)) + .Select(p => (Index: RamdiskIndex(p.Name), p.Data)).OrderBy(p => p.Index).ToList(); + if (fragments.Count > 0 && !(img.IsVendor && img.HeaderVersion >= 4)) + throw new ArgumentException("Android boot image: numbered vendor_ramdiskNN fragments exist only in a vendor_boot v4 image."); + foreach (var (index, data) in fragments) { + if (index < img.VendorRamdisks.Count) { + img.VendorRamdisks[index].Data = data; + continue; + } + if (index != img.VendorRamdisks.Count) + throw new ArgumentException($"Android vendor_boot: {AndroidBootImageMetadata.RamdiskEntryName(index)} would leave a gap after {img.VendorRamdisks.Count} fragments."); + // mkbootimg's --vendor_ramdisk becomes an unnamed platform fragment; later fragments need a name of their own. + var first = img.VendorRamdisks.Count == 0 && creating; + var name = new byte[AndroidBootImage.VendorRamdiskNameSize]; + if (!first) System.Text.Encoding.ASCII.GetBytes("ramdisk" + index.ToString("D2", CultureInfo.InvariantCulture)).CopyTo(name, 0); + img.VendorRamdisks.Add(new VendorRamdisk { + Data = data, + Type = first ? AndroidBootImage.VendorRamdiskTypePlatform : 0, + Name = name, + }); + } + } + + private static bool IsComponentName(string name) + => TryRamdiskIndex(name, out _) || name.ToLowerInvariant() is Kernel or Ramdisk or Second or RecoveryDtbo or Dtb + or BootSignature or VendorRamdisk or Bootconfig or Trailer; + + /// The rendering and whole-image view this descriptor lists; never a payload. + private static bool IsReservedName(string name) + => name.Equals(FirmwareArchive.MetadataName, StringComparison.OrdinalIgnoreCase) || name.Equals(FullImage, StringComparison.OrdinalIgnoreCase); + + private static bool TryRamdiskIndex(string name, out int index) { + index = -1; + return name.Length == VendorRamdisk.Length + 2 + && name.StartsWith(VendorRamdisk, StringComparison.OrdinalIgnoreCase) + && int.TryParse(name.AsSpan(VendorRamdisk.Length), NumberStyles.None, CultureInfo.InvariantCulture, out index); + } + + private static int RamdiskIndex(string name) => TryRamdiskIndex(name, out var i) ? i : -1; + + private static string Leaf(string name) => name.Replace('\\', '/').TrimEnd('/').Split('/')[^1]; +} diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImageMetadata.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImageMetadata.cs new file mode 100644 index 000000000..808809d8c --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImageMetadata.cs @@ -0,0 +1,215 @@ +#pragma warning disable CS1591 +using System.Globalization; +using System.Text; +using FileFormat.FirmwareCommon; + +namespace FileFormat.AndroidBootImage; + +/// +/// The metadata.ini rendering of an Android boot / vendor_boot header, and the reverse: +/// applying an edited rendering back onto an image. A key is applied only when its value differs +/// from what the image itself renders, so handing back an untouched rendering changes nothing — +/// not even the bytes past the NUL of a string field. +/// +public static class AndroidBootImageMetadata { + + public const string BootSection = "android_boot"; + public const string VendorSection = "android_vendor_boot"; + + /// Rendered for information only; editing them has no effect. + private static readonly HashSet Informational = new(StringComparer.OrdinalIgnoreCase) { + "magic", "image_id", "image_id_valid", "canonical", "canonical_issue", "trailer_size", "avb_footer", + "boot_signature_size", "parse_status", "error", + }; + + public static byte[] Render(AndroidBootImage img) { + var sb = new StringBuilder(); + void Line(string key, string value) => sb.Append(key).Append(" = ").Append(value).Append('\n'); + + sb.Append('[').Append(img.IsVendor ? VendorSection : BootSection).Append("]\n"); + Line("magic", img.IsVendor ? AndroidBootImage.VendorBootMagic : AndroidBootImage.BootMagic); + Line("header_version", Dec(img.HeaderVersion)); + if (img.IsVendor || img.HeaderVersion < 3) { + Line("page_size", Dec(img.PageSize)); + Line("kernel_load_address", FirmwareArchive.Hex(img.KernelAddress)); + Line("ramdisk_load_address", FirmwareArchive.Hex(img.RamdiskAddress)); + if (!img.IsVendor) Line("second_load_address", FirmwareArchive.Hex(img.SecondAddress)); + Line("tags_load_address", FirmwareArchive.Hex(img.TagsAddress)); + } + if (img.IsVendor || img.HeaderVersion == 2) Line("dtb_load_address", "0x" + img.DtbAddress.ToString("X16", CultureInfo.InvariantCulture)); + if (!img.IsVendor) { + Line("os_version", AndroidBootImage.FormatOsVersion(img.OsVersionPatchLevel)); + Line("os_patch_level", AndroidBootImage.FormatOsPatchLevel(img.OsVersionPatchLevel)); + } + if (img.IsVendor || img.HeaderVersion < 3) Line("board", FirmwareArchive.CString(img.Board)); + Line("cmdline", FirmwareArchive.CString(img.Cmdline)); + if (!img.IsVendor && img.HeaderVersion < 3) Line("extra_cmdline", FirmwareArchive.CString(img.ExtraCmdline)); + if (img.IsVendor || img.HeaderVersion >= 1) Line("header_size", Dec(img.HeaderSize)); + if (!img.IsVendor && img.HeaderVersion < 3) { + Line("image_id", Convert.ToHexStringLower(img.ImageId)); + Line("image_id_valid", Bool(img.ImageIdValid)); + } + if (!img.IsVendor && img.HeaderVersion >= 4) Line("boot_signature_size", Dec((uint)img.BootSignature.Length)); + Line("trailer_size", Dec((uint)img.Trailer.Length)); + Line("avb_footer", Bool(img.HasAvbFooter)); + Line("canonical", Bool(img.IsCanonical)); + if (img.NonCanonicalReason is { } reason) Line("canonical_issue", reason); + Line("parse_status", "ok"); + + if (img.IsVendor && img.HeaderVersion >= 4) + for (var i = 0; i < img.VendorRamdisks.Count; ++i) { + var r = img.VendorRamdisks[i]; + sb.Append('[').Append(RamdiskEntryName(i)).Append("]\n"); + Line("type", Dec(r.Type)); + Line("name", FirmwareArchive.CString(r.Name)); + Line("board_id", string.Join(',', r.BoardId.Select(w => FirmwareArchive.Hex(w)))); + } + return Encoding.UTF8.GetBytes(sb.ToString()); + } + + public static byte[] RenderPartial(string error) => Encoding.UTF8.GetBytes( + $"[{BootSection}]\nparse_status = partial\nerror = {error.ReplaceLineEndings(" ")}\n"); + + public static string RamdiskEntryName(int index) => "vendor_ramdisk" + index.ToString("D2", CultureInfo.InvariantCulture); + + /// + /// A fresh image of the kind and header version names (boot v0 when it + /// names none), carrying mkbootimg's defaults: base 0x10000000 with its standard offsets, + /// 2048-byte pages, and the header size of the version. + /// + public static AndroidBootImage NewImage(IReadOnlyDictionary? ini, bool vendorHint) { + var vendor = vendorHint; + if (ini is not null) { + if (ini.Keys.Any(k => k.StartsWith(VendorSection + ".", StringComparison.OrdinalIgnoreCase))) vendor = true; + else if (ini.Keys.Any(k => k.StartsWith(BootSection + ".", StringComparison.OrdinalIgnoreCase))) vendor = false; + } + var section = vendor ? VendorSection : BootSection; + uint version = vendor ? 3u : 0u; + if (ini is not null && ini.TryGetValue(section + ".header_version", out var v)) { + if (!FirmwareArchive.TryParseNumber(v, out var parsed) || parsed > 4 || vendor && parsed < 3) + throw new ArgumentException($"metadata.ini: header_version '{v}' is not valid for a {(vendor ? "vendor_boot" : "boot")} image."); + version = (uint)parsed; + } + + const uint Base = 0x10000000; + var img = new AndroidBootImage { + IsVendor = vendor, + HeaderVersion = version, + PageSize = !vendor && version >= 3 ? AndroidBootImage.V3PageSize : 2048u, + KernelAddress = Base + 0x00008000, + RamdiskAddress = Base + 0x01000000, + SecondAddress = Base + 0x00F00000, + TagsAddress = Base + 0x00000100, + DtbAddress = Base + 0x01F00000UL, + Cmdline = new byte[vendor ? AndroidBootImage.VendorArgsSize : version >= 3 ? AndroidBootImage.BootV3ArgsSize : AndroidBootImage.BootArgsSize], + HeaderSize = vendor + ? version >= 4 ? AndroidBootImage.VendorV4HeaderSize : AndroidBootImage.VendorV3HeaderSize + : version switch { + 1 => AndroidBootImage.BootV1HeaderSize, 2 => AndroidBootImage.BootV2HeaderSize, + 3 => AndroidBootImage.BootV3HeaderSize, 4 => AndroidBootImage.BootV4HeaderSize, _ => 0, + }, + }; + return img; + } + + /// Applies every key of whose value differs from the image's own rendering. + public static void Apply(AndroidBootImage img, IReadOnlyDictionary ini) { + var current = FirmwareArchive.ParseIni(Render(img)); + var section = img.IsVendor ? VendorSection : BootSection; + var other = img.IsVendor ? BootSection : VendorSection; + if (ini.Keys.Any(k => k.StartsWith(other + ".", StringComparison.OrdinalIgnoreCase))) + throw new NotSupportedException($"metadata.ini describes a {(img.IsVendor ? "boot" : "vendor_boot")} image; this is a {(img.IsVendor ? "vendor_boot" : "boot")} image."); + + foreach (var (qualified, value) in ini) { + if (current.TryGetValue(qualified, out var existing) && existing == value) continue; + var dot = qualified.IndexOf('.'); + if (dot < 0) continue; + var sec = qualified[..dot]; + var key = qualified[(dot + 1)..]; + if (Informational.Contains(key)) continue; + if (sec.Equals(section, StringComparison.OrdinalIgnoreCase)) ApplyHeaderKey(img, key, value); + else if (img.IsVendor && sec.StartsWith("vendor_ramdisk", StringComparison.OrdinalIgnoreCase) + && int.TryParse(sec.AsSpan("vendor_ramdisk".Length), NumberStyles.None, CultureInfo.InvariantCulture, out var index) + && index >= 0 && index < img.VendorRamdisks.Count) + ApplyRamdiskKey(img.VendorRamdisks[index], key, value); + } + } + + private static void ApplyHeaderKey(AndroidBootImage img, string key, string value) { + switch (key.ToLowerInvariant()) { + case "header_version": + throw new NotSupportedException("metadata.ini: the header version of an existing image cannot be changed; create a new image instead."); + case "page_size": + if (!img.IsVendor && img.HeaderVersion >= 3) return; + var page = (uint)Number(key, value, uint.MaxValue); + if (!AndroidBootImage.IsPowerOfTwoPage(page)) throw new ArgumentException($"metadata.ini: page_size {page} is not a power of two in 2048..262144."); + img.PageSize = page; + return; + case "kernel_load_address": img.KernelAddress = (uint)Number(key, value, uint.MaxValue); return; + case "ramdisk_load_address": img.RamdiskAddress = (uint)Number(key, value, uint.MaxValue); return; + case "second_load_address": img.SecondAddress = (uint)Number(key, value, uint.MaxValue); return; + case "tags_load_address": img.TagsAddress = (uint)Number(key, value, uint.MaxValue); return; + case "dtb_load_address": img.DtbAddress = Number(key, value, ulong.MaxValue); return; + case "header_size": img.HeaderSize = (uint)Number(key, value, uint.MaxValue); return; + case "os_version": + if (!AndroidBootImage.TryParseOsVersion(value, out var os)) throw new ArgumentException($"metadata.ini: os_version '{value}' is not a.b.c with each part below 128."); + img.OsVersionPatchLevel = (os << 11) | (img.OsVersionPatchLevel & 0x7FF); + return; + case "os_patch_level": + if (!AndroidBootImage.TryParseOsPatchLevel(value, out var level)) throw new ArgumentException($"metadata.ini: os_patch_level '{value}' is not YYYY-MM."); + img.OsVersionPatchLevel = (img.OsVersionPatchLevel & ~0x7FFu) | level; + return; + case "board": img.Board = Field(key, value, AndroidBootImage.BootNameSize); return; + case "cmdline": + if (!img.IsVendor && img.HeaderVersion < 3) { + // mkbootimg spills what does not fit the 512-byte field into extra_cmdline. + var bytes = Encoding.Latin1.GetBytes(value); + if (bytes.Length > AndroidBootImage.BootArgsSize - 1) { + img.Cmdline = Field(key, Encoding.Latin1.GetString(bytes, 0, AndroidBootImage.BootArgsSize - 1), AndroidBootImage.BootArgsSize); + img.ExtraCmdline = Field(key, Encoding.Latin1.GetString(bytes, AndroidBootImage.BootArgsSize - 1, bytes.Length - (AndroidBootImage.BootArgsSize - 1)), AndroidBootImage.BootExtraArgsSize); + } else + img.Cmdline = Field(key, value, AndroidBootImage.BootArgsSize); + } else + img.Cmdline = Field(key, value, img.Cmdline.Length, terminated: img.IsVendor); + return; + case "extra_cmdline": img.ExtraCmdline = Field(key, value, AndroidBootImage.BootExtraArgsSize); return; + default: + throw new ArgumentException($"metadata.ini: '{key}' is not a field of this image's header."); + } + } + + private static void ApplyRamdiskKey(VendorRamdisk r, string key, string value) { + switch (key.ToLowerInvariant()) { + case "type": r.Type = (uint)Number(key, value, uint.MaxValue); return; + case "name": r.Name = Field(key, value, AndroidBootImage.VendorRamdiskNameSize); return; + case "board_id": + var words = value.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + if (words.Length > AndroidBootImage.BoardIdWords) throw new ArgumentException($"metadata.ini: board_id has more than {AndroidBootImage.BoardIdWords} words."); + var id = new uint[AndroidBootImage.BoardIdWords]; + for (var i = 0; i < words.Length; ++i) id[i] = (uint)Number(key, words[i], uint.MaxValue); + r.BoardId = id; + return; + default: + throw new ArgumentException($"metadata.ini: '{key}' is not a field of a vendor ramdisk table entry."); + } + } + + private static ulong Number(string key, string value, ulong max) { + if (!FirmwareArchive.TryParseNumber(value, out var n) || n > max) + throw new ArgumentException($"metadata.ini: {key} '{value}' is not a number in range."); + return n; + } + + /// A NUL-padded fixed-size field. mkbootimg requires room for the terminator, except in the v3+ 1536-byte boot command line. + private static byte[] Field(string key, string value, int size, bool terminated = true) { + var bytes = Encoding.Latin1.GetBytes(value); + if (bytes.Length > size - (terminated ? 1 : 0)) + throw new ArgumentException($"metadata.ini: {key} is {bytes.Length} bytes; the field holds {size - (terminated ? 1 : 0)}."); + var field = new byte[size]; + bytes.CopyTo(field, 0); + return field; + } + + private static string Dec(uint value) => value.ToString(CultureInfo.InvariantCulture); + private static string Bool(bool value) => value ? "true" : "false"; +} diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/THIRD-PARTY-NOTICE.mkbootimg.txt b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/THIRD-PARTY-NOTICE.mkbootimg.txt new file mode 100644 index 000000000..5cd12204e --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/THIRD-PARTY-NOTICE.mkbootimg.txt @@ -0,0 +1,262 @@ +AOSP mkbootimg (Android boot image tools) — header layout converted to C# + +Source: https://android.googlesource.com/platform/system/tools/mkbootimg +Revision: d2bb0af5ba6d3198a3e99529c97eda1be0b5a093 (checked against when this notice was written) +Licences: BSD-3-Clause (include/bootimg/bootimg.h), Apache-2.0 (mkbootimg.py, unpack_bootimg.py), + both reproduced below + +AndroidBootImage and AndroidBootImageMetadata are a conversion into this codebase of the header +structures in include/bootimg/bootimg.h and of the page-aligned layout mkbootimg.py writes and +unpack_bootimg.py reads, not a verbatim copy. Magic strings, field offsets and sizes are copied +exactly, as the format requires. mkbootimg and unpack_bootimg are also run as black-box oracles in +the test suite. + +------------------------------------------------------------------------------- +include/bootimg/bootimg.h + +Copyright (C) 2007 The Android Open Source Project + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------- +mkbootimg.py (Copyright 2015, The Android Open Source Project) +unpack_bootimg.py (Copyright 2018, The Android Open Source Project) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +------------------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AndroidVbmetaFormatDescriptor.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AndroidVbmetaFormatDescriptor.cs new file mode 100644 index 000000000..d65993a24 --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AndroidVbmetaFormatDescriptor.cs @@ -0,0 +1,279 @@ +#pragma warning disable CS1591 +using System.Globalization; +using System.Text; +using Compression.Registry; +using FileFormat.FirmwareCommon; + +namespace FileFormat.AndroidVbmeta; + +/// +/// Android Verified Boot 2.0 vbmeta image (magic AVB0) as a pseudo-archive: +/// metadata.ini renders the header, the verification result and every descriptor's fields; +/// each descriptor is a member under descriptors/ as its exact encoded bytes; the public +/// key, its metadata and the signature are members of their own. +/// +/// +/// Listing verifies: the hash over header and auxiliary block is recomputed, and an RSA +/// signature is checked against the embedded public key, as avbtool verify_image does for +/// the vbmeta blob itself. +/// Writing is limited to what can be written honestly without a private key: unsigned +/// (NONE) images, created from descriptor blobs and header fields, or edited by adding, +/// replacing and removing descriptors. The bytes equal avbtool make_vbmeta_image --algorithm +/// NONE's for the same descriptors. A signed image is read and verified, never rewritten. +/// Reference: AOSP platform/external/avb (avbtool.py, libavb), MIT / +/// Apache-2.0, converted. +/// +public sealed class AndroidVbmetaFormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, + IArchiveCreatable, IArchiveModifiable, IArchiveWriteConstraints, ISyntheticEntryNames { + + public const string PublicKeyName = "public_key.bin", PublicKeyMetadataName = "public_key_metadata.bin", + SignatureName = "signature.bin", TrailerName = "trailer.bin", FullImage = "FULL.img", DescriptorFolder = "descriptors/"; + + private static readonly IReadOnlySet Synthetic = new HashSet(StringComparer.OrdinalIgnoreCase) { + FirmwareArchive.MetadataName, FullImage, + }; + + public IReadOnlySet SyntheticEntryNames => Synthetic; + public string Id => "AndroidVbmeta"; + public string DisplayName => "Android Verified Boot (vbmeta)"; + public FormatCategory Category => FormatCategory.Archive; + public FormatCapabilities Capabilities => + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | + FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; + public string DefaultExtension => ".avb"; + public IReadOnlyList Extensions => [".avb"]; + public IReadOnlyList CompoundExtensions => []; + public IReadOnlyList MagicSignatures => [new("AVB0"u8.ToArray(), Confidence: 0.9)]; + public IReadOnlyList Methods => [new("stored", "Stored")]; + public string? TarCompressionFormatId => null; + public AlgorithmFamily Family => AlgorithmFamily.Archive; + public string Description => + "Android Verified Boot 2.0 vbmeta image: header, hash/signature (verified on read) and descriptors " + + "(hash, hashtree, kernel cmdline, chain partition, property); unsigned images written as avbtool does."; + + // ── read ───────────────────────────────────────────────────────────── + + public List List(Stream stream, string? password) => FirmwareArchive.List(Entries(FirmwareArchive.ReadAll(stream))); + + public void Extract(Stream stream, string outputDir, string? password, string[]? files) + => FirmwareArchive.Extract(Entries(FirmwareArchive.ReadAll(stream)), outputDir, files); + + List IArchiveFormatOperations.ListSpan(ReadOnlySpan archive, string? password) + => FirmwareArchive.List(Entries(archive.ToArray())); + + void IArchiveFormatOperations.ExtractSpan(ReadOnlySpan archive, string outputDir, string? password, string[]? files) + => FirmwareArchive.Extract(Entries(archive.ToArray()), outputDir, files); + + public static List Entries(byte[] data) { + AvbVbmeta v; + try { + v = AvbVbmeta.Parse(data); + } catch (InvalidDataException e) { + return [ + new(FirmwareArchive.MetadataName, Encoding.UTF8.GetBytes($"[vbmeta]\nparse_status = partial\nerror = {e.Message}\n")), + new(FullImage, data), + ]; + } + var entries = new List { new(FirmwareArchive.MetadataName, RenderMetadata(v)) }; + for (var i = 0; i < v.Descriptors.Count; ++i) entries.Add(new(AvbDescriptorText.EntryName(i, v.Descriptors[i]), v.Descriptors[i].Encoded)); + if (v.PublicKey.Length > 0) entries.Add(new(PublicKeyName, v.PublicKey)); + if (v.PublicKeyMetadata.Length > 0) entries.Add(new(PublicKeyMetadataName, v.PublicKeyMetadata)); + if (v.Signature.Length > 0) entries.Add(new(SignatureName, v.Signature)); + if (v.Trailer.Length > 0) entries.Add(new(TrailerName, v.Trailer)); + return entries; + } + + public static byte[] RenderMetadata(AvbVbmeta v) { + var sb = new StringBuilder(); + void Line(string key, string value) => sb.Append(key).Append(" = ").Append(value).Append('\n'); + sb.Append("[vbmeta]\n"); + Line("required_libavb_version", $"{v.VersionMajor}.{v.VersionMinor}"); + Line("algorithm", AvbVbmeta.AlgorithmName(v.AlgorithmType)); + Line("rollback_index", v.RollbackIndex.ToString(CultureInfo.InvariantCulture)); + Line("flags", v.Flags.ToString(CultureInfo.InvariantCulture)); + Line("rollback_index_location", v.RollbackIndexLocation.ToString(CultureInfo.InvariantCulture)); + Line("release_string", AvbVbmeta.ReleaseText(v.ReleaseString)); + Line("hash", Convert.ToHexStringLower(v.Hash)); + Line("hash_valid", v.HashValid is { } h ? (h ? "true" : "false") : "n/a"); + Line("signature_valid", v.SignatureValid is { } s ? (s ? "true" : "false") : "n/a"); + if (v.PublicKey.Length > 0) Line("public_key_sha1", Convert.ToHexStringLower(System.Security.Cryptography.SHA1.HashData(v.PublicKey))); + Line("descriptors", v.Descriptors.Count.ToString(CultureInfo.InvariantCulture)); + Line("trailer_size", v.Trailer.Length.ToString(CultureInfo.InvariantCulture)); + Line("canonical", v.IsCanonical ? "true" : "false"); + if (v.NonCanonicalReason is { } reason) Line("canonical_issue", reason); + Line("parse_status", "ok"); + for (var i = 0; i < v.Descriptors.Count; ++i) { + sb.Append("[descriptor_").Append(i.ToString("D2", CultureInfo.InvariantCulture)).Append("]\n"); + AvbDescriptorText.Render(sb, v.Descriptors[i]); + } + return Encoding.UTF8.GetBytes(sb.ToString()); + } + + // ── create ─────────────────────────────────────────────────────────── + + /// + /// Writes an unsigned vbmeta image from descriptor blobs (descriptors/NN_*.bin, in + /// NN order), an optional public_key_metadata.bin, an optional trailer.bin + /// and the header fields of a metadata.ini — required libavb version, rollback index and + /// location, flags and release string. A public key or signature among the inputs is refused: + /// signing needs the private key. + /// + public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(inputs); + var v = new AvbVbmeta(); + Encoding.ASCII.GetBytes("avbtool 1.3.0").CopyTo(v.ReleaseString, 0); + var descriptors = new SortedDictionary(); + Dictionary? ini = null; + foreach (var input in inputs.Where(i => !i.IsDirectory && !string.IsNullOrEmpty(i.ArchiveName))) { + var name = input.ArchiveName.Replace('\\', '/').TrimStart('/'); + if (name.Equals(FirmwareArchive.MetadataName, StringComparison.OrdinalIgnoreCase)) ini = FirmwareArchive.ParseIni(input.ReadContent()); + else if (TryDescriptorIndex(name, out var index)) { + if (!descriptors.TryAdd(index, AvbVbmeta.FromEncoded(input.ReadContent()))) + throw new ArgumentException($"vbmeta: two descriptors claim index {index}."); + } else if (name.Equals(PublicKeyMetadataName, StringComparison.OrdinalIgnoreCase)) v.PublicKeyMetadata = input.ReadContent(); + else if (name.Equals(TrailerName, StringComparison.OrdinalIgnoreCase)) v.Trailer = input.ReadContent(); + else if (name.Equals(PublicKeyName, StringComparison.OrdinalIgnoreCase) || name.Equals(SignatureName, StringComparison.OrdinalIgnoreCase)) + throw new NotSupportedException("vbmeta: only unsigned (NONE) images can be written; signing needs the private key."); + else throw new ArgumentException($"vbmeta: '{name}' is not a vbmeta member (descriptors/NN_*.bin, public_key_metadata.bin, trailer.bin, metadata.ini)."); + } + v.Descriptors = [.. descriptors.Values]; + if (ini is not null) ApplyHeader(v, ini, creating: true); + v.Write(output); + } + + // ── write constraints ──────────────────────────────────────────────── + + /// + /// The members writes. A vbmeta holds structured AVB descriptors, not + /// files, so an arbitrary blob has nowhere to go; a public key or signature needs the private key. + /// + public bool CanAccept(ArchiveInputInfo input, out string? reason) { + ArgumentNullException.ThrowIfNull(input); + reason = null; + if (input.IsDirectory) return true; + var name = input.ArchiveName.Replace('\\', '/').TrimStart('/'); + if (name.Equals(FirmwareArchive.MetadataName, StringComparison.OrdinalIgnoreCase) || TryDescriptorIndex(name, out _) + || name.Equals(PublicKeyMetadataName, StringComparison.OrdinalIgnoreCase) || name.Equals(TrailerName, StringComparison.OrdinalIgnoreCase)) + return true; + reason = name.Equals(PublicKeyName, StringComparison.OrdinalIgnoreCase) || name.Equals(SignatureName, StringComparison.OrdinalIgnoreCase) + ? "vbmeta: only unsigned (NONE) images can be written; signing needs the private key." + : $"vbmeta: '{name}' is not a vbmeta member — {this.AcceptedInputsDescription}."; + return false; + } + + public long? MaxTotalArchiveSize => null; + + public string AcceptedInputsDescription => "accepts: descriptors/NN_*.bin, public_key_metadata.bin, trailer.bin, metadata.ini"; + + private static void ApplyHeader(AvbVbmeta v, IReadOnlyDictionary ini, bool creating) { + var current = FirmwareArchive.ParseIni(RenderMetadata(v)); + foreach (var (qualified, value) in ini) { + if (!qualified.StartsWith("vbmeta.", StringComparison.OrdinalIgnoreCase)) continue; + if (current.TryGetValue(qualified, out var existing) && existing == value) continue; + var key = qualified["vbmeta.".Length..].ToLowerInvariant(); + switch (key) { + case "required_libavb_version": { + var parts = value.Split('.'); + if (parts.Length != 2 || !uint.TryParse(parts[0], NumberStyles.None, CultureInfo.InvariantCulture, out var major) + || !uint.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var minor)) + throw new ArgumentException($"metadata.ini: required_libavb_version '{value}' is not major.minor."); + (v.VersionMajor, v.VersionMinor) = (major, minor); + break; + } + case "rollback_index": v.RollbackIndex = Number(key, value); break; + case "flags": v.Flags = checked((uint)Number(key, value)); break; + case "rollback_index_location": v.RollbackIndexLocation = checked((uint)Number(key, value)); break; + case "release_string": { + var bytes = Encoding.UTF8.GetBytes(value); + if (bytes.Length > AvbVbmeta.ReleaseStringSize - 1) throw new ArgumentException($"metadata.ini: release_string is longer than {AvbVbmeta.ReleaseStringSize - 1} bytes."); + v.ReleaseString = new byte[AvbVbmeta.ReleaseStringSize]; + bytes.CopyTo(v.ReleaseString, 0); + break; + } + case "algorithm": + if (!value.Equals("NONE", StringComparison.OrdinalIgnoreCase)) + throw new NotSupportedException($"vbmeta: algorithm {value} needs the private key; only NONE can be written."); + break; + default: + // The remaining keys (hash, verification results, counts) are renderings, not fields. + if (!creating && key is not ("hash" or "hash_valid" or "signature_valid" or "public_key_sha1" or "descriptors" or "trailer_size" or "canonical" or "canonical_issue" or "parse_status")) + throw new ArgumentException($"metadata.ini: '{key}' is not a vbmeta header field."); + break; + } + } + } + + private static ulong Number(string key, string value) + => FirmwareArchive.TryParseNumber(value, out var n) ? n : throw new ArgumentException($"metadata.ini: {key} '{value}' is not a number."); + + private static bool TryDescriptorIndex(string name, out int index) { + index = -1; + if (!name.StartsWith(DescriptorFolder, StringComparison.OrdinalIgnoreCase)) return false; + var leaf = name[DescriptorFolder.Length..]; + var underscore = leaf.IndexOf('_'); + var digits = underscore < 0 ? Path.GetFileNameWithoutExtension(leaf) : leaf[..underscore]; + return int.TryParse(digits, NumberStyles.None, CultureInfo.InvariantCulture, out index); + } + + // ── modify ─────────────────────────────────────────────────────────── + + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + var v = OpenForEdit(archive); + foreach (var input in inputs.Where(i => !i.IsDirectory && !string.IsNullOrEmpty(i.ArchiveName))) { + var name = input.ArchiveName.Replace('\\', '/').TrimStart('/'); + if (name.Equals(FirmwareArchive.MetadataName, StringComparison.OrdinalIgnoreCase)) ApplyHeader(v, FirmwareArchive.ParseIni(input.ReadContent()), creating: false); + else if (TryDescriptorIndex(name, out var index)) { + var desc = AvbVbmeta.FromEncoded(input.ReadContent()); + if (index < v.Descriptors.Count) v.Descriptors[index] = desc; + else if (index == v.Descriptors.Count) v.Descriptors.Add(desc); + else throw new ArgumentException($"vbmeta: descriptor {index} would leave a gap after {v.Descriptors.Count} descriptors."); + } else if (name.Equals(PublicKeyMetadataName, StringComparison.OrdinalIgnoreCase)) v.PublicKeyMetadata = input.ReadContent(); + else if (name.Equals(TrailerName, StringComparison.OrdinalIgnoreCase)) v.Trailer = input.ReadContent(); + else if (name.Equals(PublicKeyName, StringComparison.OrdinalIgnoreCase) || name.Equals(SignatureName, StringComparison.OrdinalIgnoreCase)) + throw new NotSupportedException("vbmeta: only unsigned (NONE) images can be written; signing needs the private key."); + else throw new ArgumentException($"vbmeta: '{name}' is not a vbmeta member."); + } + Commit(archive, v); + } + + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(archive); + var names = (entryNames ?? []).Select(n => n.Replace('\\', '/').TrimStart('/')).Where(n => !Synthetic.Contains(n)).ToList(); + if (names.Count == 0) return; + var v = OpenForEdit(archive); + var drop = new List(); + foreach (var name in names) { + if (TryDescriptorIndex(name, out var index) && index < v.Descriptors.Count) drop.Add(index); + else if (name.Equals(PublicKeyMetadataName, StringComparison.OrdinalIgnoreCase)) v.PublicKeyMetadata = []; + else if (name.Equals(TrailerName, StringComparison.OrdinalIgnoreCase)) v.Trailer = []; + } + foreach (var index in drop.Distinct().OrderDescending()) v.Descriptors.RemoveAt(index); + Commit(archive, v); + } + + public void Purge(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + var v = OpenForEdit(archive); + v.Descriptors.Clear(); + v.PublicKeyMetadata = []; + v.Trailer = []; + Commit(archive, v); + } + + private static AvbVbmeta OpenForEdit(Stream archive) { + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new ArgumentException("vbmeta editing requires a readable, writable, seekable stream.", nameof(archive)); + var v = AvbVbmeta.Parse(FirmwareArchive.ReadAll(archive)); + if (v.AlgorithmType != 0) + throw new NotSupportedException($"vbmeta: this image is signed ({AvbVbmeta.AlgorithmName(v.AlgorithmType)}); an edit would need the private key to re-sign it."); + if (!v.IsCanonical) + throw new NotSupportedException($"vbmeta: {v.NonCanonicalReason}; it is not re-laid out."); + return v; + } + + private static void Commit(Stream archive, AvbVbmeta v) + => WholeImageRebuildCommitter.Replace(archive, v.Write, candidate => _ = AvbVbmeta.Parse(FirmwareArchive.ReadAll(candidate))); +} diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbDescriptorText.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbDescriptorText.cs new file mode 100644 index 000000000..e6f95fa3b --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbDescriptorText.cs @@ -0,0 +1,109 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Globalization; +using System.Text; + +namespace FileFormat.AndroidVbmeta; + +/// +/// Human-readable renderings of AVB descriptors: the fields avbtool info_image prints, as +/// key = value lines, and a short name for the descriptor's entry. Rendering never throws — +/// a descriptor whose body does not match its declared lengths is reported as malformed. +/// +public static class AvbDescriptorText { + + private static uint U32(ReadOnlySpan e, int o) => BinaryPrimitives.ReadUInt32BigEndian(e[o..]); + private static ulong U64(ReadOnlySpan e, int o) => BinaryPrimitives.ReadUInt64BigEndian(e[o..]); + + /// The partition name, property key, or nothing — whatever names the descriptor's subject. + public static string? Subject(AvbDescriptor d) { + try { + var e = d.Encoded.AsSpan(); + return d.Tag switch { + AvbDescriptor.Hash => Utf8(e, 132, (int)U32(e, 56)), + AvbDescriptor.Hashtree => Utf8(e, 180, (int)U32(e, 104)), + AvbDescriptor.ChainPartition => Utf8(e, 92, (int)U32(e, 20)), + AvbDescriptor.Property => Utf8(e, 32, checked((int)U64(e, 16))), + _ => null, + }; + } catch (Exception ex) when (ex is ArgumentException or IndexOutOfRangeException or OverflowException) { + return null; + } + } + + public static string EntryName(int index, AvbDescriptor d) { + var subject = Subject(d); + var safe = subject is null ? "" : "_" + new string(subject.Select(c => char.IsAsciiLetterOrDigit(c) || c is '-' or '.' ? c : '_').ToArray()); + if (safe.Length > 41) safe = safe[..41]; + return $"descriptors/{index.ToString("D2", CultureInfo.InvariantCulture)}_{d.Kind}{safe}.bin"; + } + + public static void Render(StringBuilder sb, AvbDescriptor d) { + void Line(string key, string value) => sb.Append(key).Append(" = ").Append(value).Append('\n'); + Line("kind", d.Kind); + Line("size", d.Encoded.Length.ToString(CultureInfo.InvariantCulture)); + try { + var e = d.Encoded.AsSpan(); + switch (d.Tag) { + case AvbDescriptor.Hash: { + int nameLen = (int)U32(e, 56), saltLen = (int)U32(e, 60), digestLen = (int)U32(e, 64); + Line("image_size", U64(e, 16).ToString(CultureInfo.InvariantCulture)); + Line("hash_algorithm", Ascii(e.Slice(24, 32))); + Line("partition_name", Utf8(e, 132, nameLen)); + Line("salt", Hex(e, 132 + nameLen, saltLen)); + Line("digest", Hex(e, 132 + nameLen + saltLen, digestLen)); + Line("flags", U32(e, 68).ToString(CultureInfo.InvariantCulture)); + break; + } + case AvbDescriptor.Hashtree: { + int nameLen = (int)U32(e, 104), saltLen = (int)U32(e, 108), digestLen = (int)U32(e, 112); + Line("dm_verity_version", U32(e, 16).ToString(CultureInfo.InvariantCulture)); + Line("image_size", U64(e, 20).ToString(CultureInfo.InvariantCulture)); + Line("tree_offset", U64(e, 28).ToString(CultureInfo.InvariantCulture)); + Line("tree_size", U64(e, 36).ToString(CultureInfo.InvariantCulture)); + Line("data_block_size", U32(e, 44).ToString(CultureInfo.InvariantCulture)); + Line("hash_block_size", U32(e, 48).ToString(CultureInfo.InvariantCulture)); + Line("fec_num_roots", U32(e, 52).ToString(CultureInfo.InvariantCulture)); + Line("fec_offset", U64(e, 56).ToString(CultureInfo.InvariantCulture)); + Line("fec_size", U64(e, 64).ToString(CultureInfo.InvariantCulture)); + Line("hash_algorithm", Ascii(e.Slice(72, 32))); + Line("partition_name", Utf8(e, 180, nameLen)); + Line("salt", Hex(e, 180 + nameLen, saltLen)); + Line("root_digest", Hex(e, 180 + nameLen + saltLen, digestLen)); + Line("flags", U32(e, 116).ToString(CultureInfo.InvariantCulture)); + break; + } + case AvbDescriptor.KernelCmdline: + Line("flags", U32(e, 16).ToString(CultureInfo.InvariantCulture)); + Line("cmdline", Utf8(e, 24, (int)U32(e, 20))); + break; + case AvbDescriptor.ChainPartition: { + int nameLen = (int)U32(e, 20), keyLen = (int)U32(e, 24); + Line("rollback_index_location", U32(e, 16).ToString(CultureInfo.InvariantCulture)); + Line("partition_name", Utf8(e, 92, nameLen)); + Line("public_key_sha1", Convert.ToHexStringLower(System.Security.Cryptography.SHA1.HashData(e.Slice(92 + nameLen, keyLen)))); + Line("flags", U32(e, 28).ToString(CultureInfo.InvariantCulture)); + break; + } + case AvbDescriptor.Property: { + var keyLen = checked((int)U64(e, 16)); + var valueLen = checked((int)U64(e, 24)); + Line("key", Utf8(e, 32, keyLen)); + var value = e.Slice(32 + keyLen + 1, valueLen); + Line("value", value.ContainsAnyExceptInRange((byte)0x20, (byte)0x7E) ? "hex:" + Convert.ToHexStringLower(value) : Encoding.ASCII.GetString(value)); + break; + } + } + } catch (Exception ex) when (ex is ArgumentException or IndexOutOfRangeException or OverflowException) { + Line("malformed", "true"); + } + } + + private static string Utf8(ReadOnlySpan e, int offset, int length) => Encoding.UTF8.GetString(e.Slice(offset, length)).ReplaceLineEndings(" "); + private static string Hex(ReadOnlySpan e, int offset, int length) => Convert.ToHexStringLower(e.Slice(offset, length)); + + private static string Ascii(ReadOnlySpan field) { + var end = field.IndexOf((byte)0); + return Encoding.ASCII.GetString(end < 0 ? field : field[..end]); + } +} diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbVbmeta.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbVbmeta.cs new file mode 100644 index 000000000..bc4f9f6a0 --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbVbmeta.cs @@ -0,0 +1,241 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace FileFormat.AndroidVbmeta; + +/// One AVB descriptor, kept as its exact encoded bytes (16-byte tag/length header, body, padding). +public sealed record AvbDescriptor(ulong Tag, byte[] Encoded) { + public const ulong Property = 0, Hashtree = 1, Hash = 2, KernelCmdline = 3, ChainPartition = 4; + + public string Kind => this.Tag switch { + Property => "property", Hashtree => "hashtree", Hash => "hash", KernelCmdline => "kernel_cmdline", + ChainPartition => "chain_partition", _ => "tag" + this.Tag.ToString(CultureInfo.InvariantCulture), + }; + + public ReadOnlySpan Body => this.Encoded.AsSpan(16); +} + +/// +/// An Android Verified Boot 2.0 vbmeta image (magic AVB0): the 256-byte header, the +/// authentication block (hash, signature) and the auxiliary block (descriptors, public key, public +/// key metadata), parsed and verified, and written back the way avbtool make_vbmeta_image +/// lays one out. +/// +/// +/// Structures follow AOSP platform/external/avb — libavb/avb_vbmeta_image.h, +/// avb_descriptor.h and avbtool.py (MIT / Apache-2.0), converted. All integers are +/// big-endian. The hash covers the header and the auxiliary block; the signature is RSA PKCS#1 v1.5 +/// over the same bytes, with the public key in AVB's own encoding (bit count, n0inv, modulus, R²). +/// +public sealed class AvbVbmeta { + + public const int HeaderSize = 256; + public const int ReleaseStringSize = 48; + public const int ReservedSize = 80; + + public uint VersionMajor { get; set; } = 1; + public uint VersionMinor { get; set; } + public uint AlgorithmType { get; set; } + public ulong RollbackIndex { get; set; } + public uint Flags { get; set; } + public uint RollbackIndexLocation { get; set; } + /// The 48-byte release-string field, raw (avbtool writes 47 bytes plus a NUL). + public byte[] ReleaseString { get; set; } = new byte[ReleaseStringSize]; + public byte[] Reserved { get; set; } = new byte[ReservedSize]; + + public List Descriptors { get; set; } = []; + public byte[] PublicKey { get; set; } = []; + public byte[] PublicKeyMetadata { get; set; } = []; + public byte[] Hash { get; set; } = []; + public byte[] Signature { get; set; } = []; + /// Bytes after the vbmeta blob (partition padding). + public byte[] Trailer { get; set; } = []; + + /// True when the image is laid out as avbtool lays it out, so a rewrite reproduces it. + public bool IsCanonical { get; private set; } = true; + public string? NonCanonicalReason { get; private set; } + /// Null when the algorithm carries no hash. + public bool? HashValid { get; private set; } + /// Null when the algorithm carries no signature this reader can check. + public bool? SignatureValid { get; private set; } + + public static string AlgorithmName(uint type) => type switch { + 0 => "NONE", 1 => "SHA256_RSA2048", 2 => "SHA256_RSA4096", 3 => "SHA256_RSA8192", + 4 => "SHA512_RSA2048", 5 => "SHA512_RSA4096", 6 => "SHA512_RSA8192", 7 => "MLDSA65", 8 => "MLDSA87", + _ => "UNKNOWN", + }; + + // ── parse ──────────────────────────────────────────────────────────── + + private static uint U32(ReadOnlySpan d, int o) => BinaryPrimitives.ReadUInt32BigEndian(d[o..]); + private static ulong U64(ReadOnlySpan d, int o) => BinaryPrimitives.ReadUInt64BigEndian(d[o..]); + + public static AvbVbmeta Parse(ReadOnlySpan d) { + if (d.Length < HeaderSize) throw new InvalidDataException($"vbmeta: {d.Length} bytes is shorter than the {HeaderSize}-byte header."); + if (!d[..4].SequenceEqual("AVB0"u8)) throw new InvalidDataException("vbmeta: no AVB0 magic."); + var v = new AvbVbmeta { + VersionMajor = U32(d, 4), + VersionMinor = U32(d, 8), + AlgorithmType = U32(d, 28), + RollbackIndex = U64(d, 112), + Flags = U32(d, 120), + RollbackIndexLocation = U32(d, 124), + ReleaseString = d.Slice(128, ReleaseStringSize).ToArray(), + Reserved = d.Slice(176, ReservedSize).ToArray(), + }; + var authSize = U64(d, 12); + var auxSize = U64(d, 20); + if (authSize > int.MaxValue || auxSize > int.MaxValue || HeaderSize + (long)authSize + (long)auxSize > d.Length) + throw new InvalidDataException($"vbmeta: blocks of {authSize} + {auxSize} bytes run past the end of the {d.Length}-byte image."); + var auth = d.Slice(HeaderSize, (int)authSize); + var aux = d.Slice(HeaderSize + (int)authSize, (int)auxSize); + + v.Hash = Slice(auth, U64(d, 32), U64(d, 40), "hash"); + v.Signature = Slice(auth, U64(d, 48), U64(d, 56), "signature"); + v.PublicKey = Slice(aux, U64(d, 64), U64(d, 72), "public key"); + v.PublicKeyMetadata = Slice(aux, U64(d, 80), U64(d, 88), "public key metadata"); + var descOffset = U64(d, 96); + var descSize = U64(d, 104); + var descriptors = Slice(aux, descOffset, descSize, "descriptors"); + for (var o = 0; o < descriptors.Length;) { + if (descriptors.Length - o < 16) throw new InvalidDataException("vbmeta: a descriptor header is cut short."); + var tag = U64(descriptors, o); + var following = U64(descriptors, o + 8); + if (following > (ulong)(descriptors.Length - o - 16) || following % 8 != 0) + throw new InvalidDataException($"vbmeta: descriptor at offset {o} declares {following} bytes that do not fit."); + v.Descriptors.Add(new AvbDescriptor(tag, descriptors.AsSpan(o, 16 + (int)following).ToArray())); + o += 16 + (int)following; + } + v.Trailer = d[(HeaderSize + (int)authSize + (int)auxSize)..].ToArray(); + + // avbtool's layout: hash then signature; descriptors, key, key metadata; both blocks padded to 64 with zeros. + var keyLen = v.PublicKey.Length; + if (U64(d, 32) != 0 || U64(d, 48) != (ulong)v.Hash.Length + || descOffset != 0 || U64(d, 64) != descSize || U64(d, 80) != descSize + (ulong)keyLen + || authSize != Round64((ulong)(v.Hash.Length + v.Signature.Length)) + || auxSize != Round64(descSize + (ulong)keyLen + (ulong)v.PublicKeyMetadata.Length) + || auth[(v.Hash.Length + v.Signature.Length)..].ContainsAnyExcept((byte)0) + || aux[(int)(descSize + (ulong)keyLen + (ulong)v.PublicKeyMetadata.Length)..].ContainsAnyExcept((byte)0)) + v.MarkNonCanonical("the blocks are not laid out as avbtool lays them out"); + v.Verify(d[..HeaderSize], aux); + return v; + } + + private static byte[] Slice(ReadOnlySpan block, ulong offset, ulong size, string what) { + if (offset > (ulong)block.Length || size > (ulong)block.Length - offset) + throw new InvalidDataException($"vbmeta: the {what} ({size} bytes at {offset}) runs past its {block.Length}-byte block."); + return block.Slice((int)offset, (int)size).ToArray(); + } + + private static ulong Round64(ulong n) => (n + 63) / 64 * 64; + + private void MarkNonCanonical(string reason) { + this.IsCanonical = false; + this.NonCanonicalReason ??= reason; + } + + // ── verify ─────────────────────────────────────────────────────────── + + private static HashAlgorithmName? HashOf(uint type) => type switch { + 1 or 2 or 3 => HashAlgorithmName.SHA256, + 4 or 5 or 6 => HashAlgorithmName.SHA512, + _ => null, + }; + + private void Verify(ReadOnlySpan header, ReadOnlySpan aux) { + if (HashOf(this.AlgorithmType) is not { } hashName) return; + var message = new byte[header.Length + aux.Length]; + header.CopyTo(message); + aux.CopyTo(message.AsSpan(header.Length)); + var digest = hashName == HashAlgorithmName.SHA256 ? SHA256.HashData(message) : SHA512.HashData(message); + this.HashValid = digest.AsSpan().SequenceEqual(this.Hash); + if (TryRsaKey(this.PublicKey) is not { } parameters) { + this.SignatureValid = false; + return; + } + try { + using var rsa = RSA.Create(parameters); + this.SignatureValid = this.HashValid == true && rsa.VerifyData(message, this.Signature, hashName, RSASignaturePadding.Pkcs1); + } catch (CryptographicException) { + this.SignatureValid = false; + } + } + + /// AVB's public key encoding: big-endian bit count and n0inv, then the modulus and R², each bit-count/8 bytes. Exponent 65537. + public static RSAParameters? TryRsaKey(ReadOnlySpan key) { + if (key.Length < 8) return null; + var bits = BinaryPrimitives.ReadUInt32BigEndian(key); + if (bits is not (2048 or 4096 or 8192) || key.Length != 8 + 2 * (bits / 8)) return null; + return new RSAParameters { Modulus = key.Slice(8, (int)bits / 8).ToArray(), Exponent = [1, 0, 1] }; + } + + // ── write ──────────────────────────────────────────────────────────── + + /// + /// Writes the vbmeta blob as avbtool's _generate_vbmeta_blob does, then the trailer. Only + /// the unsigned NONE algorithm can be written: anything else needs the private key. + /// + public void Write(Stream output) { + ArgumentNullException.ThrowIfNull(output); + if (this.AlgorithmType != 0 || this.Hash.Length > 0 || this.Signature.Length > 0 || this.PublicKey.Length > 0) + throw new NotSupportedException("vbmeta: only the unsigned NONE algorithm can be written; signing needs the private key."); + var descriptors = new MemoryStream(); + foreach (var desc in this.Descriptors) { + Validate(desc); + descriptors.Write(desc.Encoded); + } + var descSize = (ulong)descriptors.Length; + var auxSize = Round64(descSize + (ulong)this.PublicKeyMetadata.Length); + + var h = new byte[HeaderSize]; + "AVB0"u8.CopyTo(h); + BinaryPrimitives.WriteUInt32BigEndian(h.AsSpan(4), this.VersionMajor); + BinaryPrimitives.WriteUInt32BigEndian(h.AsSpan(8), this.VersionMinor); + BinaryPrimitives.WriteUInt64BigEndian(h.AsSpan(12), 0); // authentication block: no hash, no signature + BinaryPrimitives.WriteUInt64BigEndian(h.AsSpan(20), auxSize); + BinaryPrimitives.WriteUInt32BigEndian(h.AsSpan(28), 0); + // hash 32/40 and signature 48/56 stay zero + BinaryPrimitives.WriteUInt64BigEndian(h.AsSpan(64), descSize); // public key offset (size 0) + BinaryPrimitives.WriteUInt64BigEndian(h.AsSpan(80), descSize); // public key metadata offset + BinaryPrimitives.WriteUInt64BigEndian(h.AsSpan(88), (ulong)this.PublicKeyMetadata.Length); + BinaryPrimitives.WriteUInt64BigEndian(h.AsSpan(96), 0); + BinaryPrimitives.WriteUInt64BigEndian(h.AsSpan(104), descSize); + BinaryPrimitives.WriteUInt64BigEndian(h.AsSpan(112), this.RollbackIndex); + BinaryPrimitives.WriteUInt32BigEndian(h.AsSpan(120), this.Flags); + BinaryPrimitives.WriteUInt32BigEndian(h.AsSpan(124), this.RollbackIndexLocation); + this.ReleaseString.AsSpan(0, Math.Min(ReleaseStringSize, this.ReleaseString.Length)).CopyTo(h.AsSpan(128)); + h[128 + ReleaseStringSize - 1] = 0; + this.Reserved.AsSpan(0, Math.Min(ReservedSize, this.Reserved.Length)).CopyTo(h.AsSpan(176)); + + output.Write(h); + descriptors.Position = 0; + descriptors.CopyTo(output); + output.Write(this.PublicKeyMetadata); + output.Write(new byte[auxSize - descSize - (ulong)this.PublicKeyMetadata.Length]); + output.Write(this.Trailer); + } + + /// Checks an encoded descriptor's own length field, so a malformed blob is refused rather than written. + public static void Validate(AvbDescriptor desc) { + var e = desc.Encoded; + if (e.Length < 16 || (e.Length - 16) % 8 != 0 + || BinaryPrimitives.ReadUInt64BigEndian(e.AsSpan(8)) != (ulong)(e.Length - 16) + || BinaryPrimitives.ReadUInt64BigEndian(e) != desc.Tag) + throw new ArgumentException($"vbmeta: a {desc.Kind} descriptor of {e.Length} bytes is not a well-formed AVB descriptor (tag, length, 8-byte padded body)."); + } + + public static AvbDescriptor FromEncoded(byte[] encoded) { + if (encoded.Length < 16) throw new ArgumentException("vbmeta: a descriptor is at least 16 bytes."); + var desc = new AvbDescriptor(BinaryPrimitives.ReadUInt64BigEndian(encoded), encoded); + Validate(desc); + return desc; + } + + public static string ReleaseText(byte[] field) { + var end = Array.IndexOf(field, (byte)0); + return Encoding.UTF8.GetString(field, 0, end < 0 ? field.Length : end); + } +} diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/THIRD-PARTY-NOTICE.avb.txt b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/THIRD-PARTY-NOTICE.avb.txt new file mode 100644 index 000000000..452bf67c3 --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/THIRD-PARTY-NOTICE.avb.txt @@ -0,0 +1,32 @@ +AOSP Android Verified Boot (libavb / avbtool) — vbmeta structures converted to C# + +Source: https://android.googlesource.com/platform/external/avb +Revision: c5066a96caa7bf4150c0a8cc8cc14ab81733fdc7 +Licence: MIT (the licence of libavb/avb_vbmeta_image.h, libavb/avb_descriptor.h and avbtool.py; + reproduced below) + +AvbVbmeta and AvbDescriptorText are a conversion into this codebase of the vbmeta header, the +descriptor structures and the public-key encoding defined in those files, not a verbatim copy. +Magic strings, field offsets, descriptor tags and algorithm numbers are copied exactly, as the +format requires. avbtool at the same revision is also run as a black-box oracle in the test suite. + +Copyright (C) 2016 The Android Open Source Project + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareCommon/EmbeddedPayload.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareCommon/EmbeddedPayload.cs new file mode 100644 index 000000000..722d9be8b --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareCommon/EmbeddedPayload.cs @@ -0,0 +1,89 @@ +#pragma warning disable CS1591 +using Compression.Registry; + +namespace FileFormat.FirmwareCommon; + +/// +/// Recognises and unpacks a compressed payload embedded in a firmware image by the magic of its +/// stream format, using this repository's own stream codecs through the registry. A payload is +/// only reported as decompressed when the codec decodes it without error, so nothing is ever +/// guessed into existence. +/// +public static class EmbeddedPayload { + + /// Stream formats a kernel or firmware payload is commonly compressed with: registry id, file extension, magic. + private static readonly (string Id, string Extension, byte[] Magic)[] Known = [ + ("Gzip", "gz", [0x1F, 0x8B, 0x08]), + ("Xz", "xz", [0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00]), + ("Bzip2", "bz2", [0x42, 0x5A, 0x68]), + ("Zstd", "zst", [0x28, 0xB5, 0x2F, 0xFD]), + ("Lz4", "lz4", [0x02, 0x21, 0x4C, 0x18]), + ("Lzop", "lzo", [0x89, 0x4C, 0x5A, 0x4F, 0x00, 0x0D, 0x0A, 0x1A, 0x0A]), + ("Lzma", "lzma", [0x5D, 0x00, 0x00]), + ]; + + /// The stream format whose magic starts , or null. + public static (string Id, string Extension)? Identify(ReadOnlySpan data) { + foreach (var (id, extension, magic) in Known) + if (data.StartsWith(magic)) return (id, extension); + return null; + } + + /// + /// Decodes with the codec names, giving up + /// past output bytes. Null when the codec is not registered or rejects + /// the stream. + /// + public static byte[]? TryDecompress(string formatId, ReadOnlyMemory data, long limit = 1L << 30) { + if (FormatRegistry.GetStreamOps(formatId) is not { } ops) return null; + // A stray 5D 00 00 is no LZMA stream: refuse property bytes past lc/lp/pb's range and dictionaries + // a kernel compressor never uses, before the decoder allocates a window for them. + if (formatId == "Lzma") { + var span = data.Span; + if (span.Length < 13 || span[0] >= 9 * 5 * 5 || System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(span[1..]) > 1u << 27) + return null; + } + try { + using var input = new MemoryStream(data.ToArray(), writable: false); + using var output = new CappedStream(limit); + ops.Decompress(input, output); + return output.Length == 0 ? null : output.ToArray(); + } catch (Exception e) when (e is InvalidDataException or IOException or ArgumentException or InvalidOperationException + or IndexOutOfRangeException or NotSupportedException or OverflowException or EndOfStreamException) { + return null; + } + } + + /// + /// Scans from for the first known stream magic + /// whose codec decodes and whose output satisfies . + /// + public static (int Offset, string Id, string Extension, byte[] Decoded)? Find(ReadOnlyMemory data, int start, Func accept, int maxCandidates = 64) { + var span = data.Span; + var tried = 0; + for (var i = Math.Max(0, start); i < span.Length - 4 && tried < maxCandidates; ++i) { + if (Identify(span[i..]) is not { } kind) continue; + ++tried; + if (TryDecompress(kind.Id, data[i..]) is { } decoded && accept(decoded)) return (i, kind.Id, kind.Extension, decoded); + } + return null; + } + + /// A memory stream that refuses to grow past a cap, so a hostile stream cannot exhaust memory. + private sealed class CappedStream(long cap) : MemoryStream { + public override void Write(byte[] buffer, int offset, int count) { + if (this.Length + count > cap) throw new InvalidDataException($"decoded payload exceeds {cap} bytes"); + base.Write(buffer, offset, count); + } + + public override void Write(ReadOnlySpan buffer) { + if (this.Length + buffer.Length > cap) throw new InvalidDataException($"decoded payload exceeds {cap} bytes"); + base.Write(buffer); + } + + public override void WriteByte(byte value) { + if (this.Length + 1 > cap) throw new InvalidDataException($"decoded payload exceeds {cap} bytes"); + base.WriteByte(value); + } + } +} diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareCommon/FirmwareArchive.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareCommon/FirmwareArchive.cs new file mode 100644 index 000000000..5cb8524c8 --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareCommon/FirmwareArchive.cs @@ -0,0 +1,82 @@ +#pragma warning disable CS1591 +using System.Globalization; +using System.Text; +using Compression.Registry; +using static Compression.Registry.FormatHelpers; + +namespace FileFormat.FirmwareCommon; + +/// One member a firmware container exposes: a slice of the image or a rendering of it. +public sealed record FirmwareEntry(string Name, ReadOnlyMemory Data); + +/// +/// The list / extract plumbing every firmware pseudo-archive shares: a firmware image is +/// parsed into an ordered set of named slices, and listing and extracting both walk that set. +/// +public static class FirmwareArchive { + + public const string MetadataName = "metadata.ini"; + + public static byte[] ReadAll(Stream stream) { + ArgumentNullException.ThrowIfNull(stream); + if (stream.CanSeek) stream.Position = 0; + using var ms = new MemoryStream(); + stream.CopyTo(ms); + return ms.ToArray(); + } + + public static List List(IReadOnlyList entries) + => entries.Select((e, i) => new ArchiveEntryInfo(i, e.Name, e.Data.Length, e.Data.Length, + "stored", false, false, null)).ToList(); + + public static void Extract(IReadOnlyList entries, string outputDir, string[]? files) { + foreach (var e in entries) { + if (files is { Length: > 0 } && !MatchesFilter(e.Name, files)) continue; + using var target = CreateEntryFile(outputDir, e.Name); + target.Write(e.Data.Span); + } + } + + /// A NUL-terminated field as text: everything before the first NUL, Latin-1 so no byte is lost. + public static string CString(ReadOnlySpan field) { + var end = field.IndexOf((byte)0); + return Encoding.Latin1.GetString(end < 0 ? field : field[..end]); + } + + public static string Hex(ulong value) => "0x" + value.ToString("X8", CultureInfo.InvariantCulture); + + /// + /// Parses the key = value lines of an INI rendering into section-qualified keys + /// (section.key); keys before any section are unqualified. The value keeps its + /// exact text after the single blank that follows the =. + /// + public static Dictionary ParseIni(ReadOnlySpan ini) { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + var section = ""; + foreach (var raw in Encoding.UTF8.GetString(ini).Split('\n')) { + var line = raw.TrimEnd('\r'); + var trimmed = line.Trim(); + if (trimmed.Length == 0 || trimmed[0] is ';' or '#') continue; + if (trimmed[0] == '[' && trimmed[^1] == ']') { + section = trimmed[1..^1].Trim(); + continue; + } + var eq = line.IndexOf('='); + if (eq <= 0) continue; + var key = line[..eq].Trim(); + var value = line[(eq + 1)..]; + if (value.StartsWith(' ')) value = value[1..]; + result[section.Length == 0 ? key : section + "." + key] = value; + } + return result; + } + + public static bool TryParseNumber(string text, out ulong value) { + text = text.Trim(); + var space = text.IndexOf(' '); + if (space > 0) text = text[..space]; + return text.StartsWith("0x", StringComparison.OrdinalIgnoreCase) + ? ulong.TryParse(text.AsSpan(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out value) + : ulong.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out value); + } +} diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareCommon/FirmwareCrc32.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareCommon/FirmwareCrc32.cs new file mode 100644 index 000000000..5916e059a --- /dev/null +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareCommon/FirmwareCrc32.cs @@ -0,0 +1,30 @@ +#pragma warning disable CS1591 +namespace FileFormat.FirmwareCommon; + +/// +/// The reflected CRC-32 (polynomial 0xEDB88320) firmware headers use, exposed raw: initial value +/// and final XOR are the caller's, because TRX for one stores the register without the final +/// inversion. +/// +public static class FirmwareCrc32 { + + private static readonly uint[] Table = BuildTable(); + + private static uint[] BuildTable() { + var table = new uint[256]; + for (uint i = 0; i < 256; ++i) { + var c = i; + for (var k = 0; k < 8; ++k) c = (c & 1) != 0 ? 0xEDB88320u ^ (c >> 1) : c >> 1; + table[i] = c; + } + return table; + } + + public static uint Update(uint crc, ReadOnlySpan data) { + foreach (var b in data) crc = Table[(crc ^ b) & 0xFF] ^ (crc >> 8); + return crc; + } + + /// The standard CRC-32 (zlib, IEEE 802.3). + public static uint Compute(ReadOnlySpan data) => ~Update(0xFFFFFFFFu, data); +} diff --git a/Hawkynt.FileFormats.FileSystems/README.md b/Hawkynt.FileFormats.FileSystems/README.md index 264c19eb5..6bee1a597 100644 --- a/Hawkynt.FileFormats.FileSystems/README.md +++ b/Hawkynt.FileFormats.FileSystems/README.md @@ -59,6 +59,8 @@ The `Id`, `State` and verb columns are read off the descriptors by `FilesystemRe | Format | Id | State | Compact | Defrag | Sort | Wipe | Shrink | Geometry | Purge | Proof | Notes | Reference | | --- | --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | --- | --- | --- | +| [Android boot image](https://source.android.com/docs/core/architecture/bootloader/boot-image-header) | `AndroidBootImage` | R/W | — | — | — | — | — | — | ✅ | [`mkbootimg` / `unpack_bootimg`](https://android.googlesource.com/platform/system/tools/mkbootimg) — byte-identical both ways | boot/recovery header v0–v4 and vendor_boot v3/v4; components as `unpack_bootimg` names them, header in `metadata.ini`; edits refuse a signed image (v4 boot signature, AVB footer) and layouts `mkbootimg` cannot produce | [Boot image header](https://source.android.com/docs/core/architecture/bootloader/boot-image-header) | +| [Android Verified Boot (vbmeta)](https://source.android.com/docs/security/features/verifiedboot/avb) | `AndroidVbmeta` | R/W | — | — | — | — | — | — | ✅ | [`avbtool`](https://android.googlesource.com/platform/external/avb) — byte-identical unsigned images, `info_image` reads ours; signatures checked against avbtool-signed vectors | hash and RSA signature verified on read; each descriptor a member; only unsigned (`NONE`) images are written or edited — a signed image is read-only, since re-signing needs the private key | [AVB README](https://android.googlesource.com/platform/external/avb/+/main/README.md) | | [Apple IPSW](https://en.wikipedia.org/wiki/IPSW) | `Ipsw` | R/W | ✅ | ✅ rebuild | — | ✅ | ✅ | — | ✅ | own reader + struct-parity tests | ZIP-based firmware archive | [IPSW](https://en.wikipedia.org/wiki/IPSW) | | [BIN/CUE](https://en.wikipedia.org/wiki/Cue_sheet_(computing)) | `BinCue` | R/W | — | — | — | ✅ | — | — | ✅ | own reader + struct-parity tests | Sector-image editor | [CUE sheet](https://wiki.hydrogenaud.io/index.php?title=Cue_sheet) | | [CDI](https://en.wikipedia.org/wiki/DiscJuggler) | `Cdi` | R/W | ✅ | ✅ rebuild | — | — | ✅ | — | ✅ | own reader + struct-parity tests | ISO 9660 edits rebuild; low-level existing-sector rewrite remains available | [DiscJuggler](https://en.wikipedia.org/wiki/DiscJuggler) | @@ -282,7 +284,7 @@ ops.Remove(image, ["var/log/old.log"]); -Every public and protected member of all 874 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.FileSystems/REFERENCE.md). +Every public and protected member of all 886 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.FileFormats.FileSystems/REFERENCE.md). diff --git a/Hawkynt.FileFormats.FileSystems/REFERENCE.md b/Hawkynt.FileFormats.FileSystems/REFERENCE.md index 7a78b598b..5872c72bb 100644 --- a/Hawkynt.FileFormats.FileSystems/REFERENCE.md +++ b/Hawkynt.FileFormats.FileSystems/REFERENCE.md @@ -7,6 +7,251 @@ > Every public and protected type and member, read from the built assembly and merged > with its XML documentation. Generated — edit the XML docs in source, not this file. +### Namespace `FileFormat.AndroidBootImage` + +[`AndroidBootImage`](#androidbootimage) · [`AndroidBootImageFormatDescriptor`](#androidbootimageformatdescriptor) · [`AndroidBootImageMetadata`](#androidbootimagemetadata) · [`VendorRamdisk`](#vendorramdisk) + +#### `AndroidBootImage` + +An Android `boot.img` / `recovery.img` (magic `ANDROID!`, header v0–v4) or `vendor_boot.img` (magic `VNDRBOOT`, header v3–v4), parsed into its header fields and page-aligned components, and written back the way AOSP `mkbootimg` lays one out. + +| Member | Signature | Summary | +| --- | --- | --- | +| `AndroidBootImage` | `AndroidBootImage()` | | +| `BoardIdWords` | `const int BoardIdWords` | | +| `BootArgsSize` | `const int BootArgsSize` | | +| `BootExtraArgsSize` | `const int BootExtraArgsSize` | | +| `BootMagic` | `const string BootMagic` | | +| `BootNameSize` | `const int BootNameSize` | | +| `BootV1HeaderSize` | `const uint BootV1HeaderSize` | Header lengths per version, as `mkbootimg` writes them into the header-size field. | +| `BootV2HeaderSize` | `const uint BootV2HeaderSize` | Header lengths per version, as `mkbootimg` writes them into the header-size field. | +| `BootV3ArgsSize` | `const int BootV3ArgsSize` | | +| `BootV3HeaderSize` | `const uint BootV3HeaderSize` | Header lengths per version, as `mkbootimg` writes them into the header-size field. | +| `BootV4HeaderSize` | `const uint BootV4HeaderSize` | Header lengths per version, as `mkbootimg` writes them into the header-size field. | +| `V3PageSize` | `const int V3PageSize` | | +| `VendorArgsSize` | `const int VendorArgsSize` | | +| `VendorBootMagic` | `const string VendorBootMagic` | | +| `VendorRamdiskEntrySize` | `const int VendorRamdiskEntrySize` | | +| `VendorRamdiskNameSize` | `const int VendorRamdiskNameSize` | | +| `VendorRamdiskTypePlatform` | `const uint VendorRamdiskTypePlatform` | | +| `VendorV3HeaderSize` | `const uint VendorV3HeaderSize` | | +| `VendorV4HeaderSize` | `const uint VendorV4HeaderSize` | | +| `Board` | `byte[] Board { get; set; }` | | +| `BootSignature` | `byte[] BootSignature { get; set; }` | | +| `Bootconfig` | `byte[] Bootconfig { get; set; }` | | +| `Cmdline` | `byte[] Cmdline { get; set; }` | The command-line field, raw: 512 bytes (boot v0–v2), 1536 (boot v3/v4), 2048 (vendor). | +| `DtbAddress` | `ulong DtbAddress { get; set; }` | | +| `Dtb` | `byte[] Dtb { get; set; }` | | +| `EffectivePageSize` | `uint EffectivePageSize { get; }` | | +| `ExtraCmdline` | `byte[] ExtraCmdline { get; set; }` | | +| `HasAvbFooter` | `bool HasAvbFooter { get; }` | True when the trailer ends in an AVB footer (`AVBf` in the last 64 bytes). | +| `HeaderSize` | `uint HeaderSize { get; set; }` | The header-size field (boot v1+, vendor); 0 where the version has none. | +| `HeaderTail` | `byte[] HeaderTail { get; set; }` | Header-region bytes after the fields this parser knows, up to the first component. | +| `HeaderVersion` | `uint HeaderVersion { get; set; }` | | +| `ImageIdValid` | `bool ImageIdValid { get; }` | True when the stored image id equals the SHA-1 `mkbootimg` computes (boot v0–v2). | +| `ImageId` | `byte[] ImageId { get; set; }` | | +| `IsCanonical` | `bool IsCanonical { get; }` | | +| `IsVendor` | `bool IsVendor { get; set; }` | | +| `KernelAddress` | `uint KernelAddress { get; set; }` | | +| `Kernel` | `byte[] Kernel { get; set; }` | | +| `KnownHeaderLength` | `int KnownHeaderLength { get; }` | | +| `NonCanonicalReason` | `string NonCanonicalReason { get; }` | | +| `OsVersionPatchLevel` | `uint OsVersionPatchLevel { get; set; }` | `(os_version << 11) \| os_patch_level`, raw. | +| `PageSize` | `uint PageSize { get; set; }` | Flash page size. Fixed at 4096 for boot v3/v4. | +| `RamdiskAddress` | `uint RamdiskAddress { get; set; }` | | +| `Ramdisk` | `byte[] Ramdisk { get; set; }` | | +| `RecoveryDtbo` | `byte[] RecoveryDtbo { get; set; }` | | +| `Reserved` | `uint[] Reserved { get; set; }` | | +| `SecondAddress` | `uint SecondAddress { get; set; }` | | +| `Second` | `byte[] Second { get; set; }` | | +| `TagsAddress` | `uint TagsAddress { get; set; }` | | +| `Trailer` | `byte[] Trailer { get; set; }` | Bytes after the last component: partition padding, an AVB footer, anything appended. | +| `VendorRamdisks` | `List VendorRamdisks { get; set; }` | | +| `ComputeImageId` | `byte[] ComputeImageId()` | The boot v0–v2 image id exactly as `mkbootimg` computes it: SHA-1 over each component followed by its 32-bit length (kernel, ramdisk, second, then recovery DTBO from v1 and DTB from v2), zero-extended to 32 bytes. | +| `FormatOsPatchLevel` | `static string FormatOsPatchLevel(uint packed)` | | +| `FormatOsVersion` | `static string FormatOsVersion(uint packed)` | | +| `IsPowerOfTwoPage` | `static bool IsPowerOfTwoPage(uint page)` | | +| `Pages` | `static int Pages(long size, int page)` | | +| `Parse` | `static AndroidBootImage Parse(ReadOnlySpan data)` | | +| `TryParseOsPatchLevel` | `static bool TryParseOsPatchLevel(string text, out uint level)` | Parses `YYYY-MM[-DD]` as `mkbootimg --os_patch_level` does (7 bits of year past 2000, 4 of month). | +| `TryParseOsVersion` | `static bool TryParseOsVersion(string text, out uint version)` | Parses `a[.b[.c]]`, 7 bits per field, as `mkbootimg --os_version` does. | +| `Write` | `void Write(Stream output, bool recomputeImageId)` | Writes the image in `mkbootimg` order: header page(s), then each component padded to the page size. A boot v0–v2 image id is recomputed when `recomputeImageId` is set (a fresh image, or one whose stored id was valid), else kept as it was. | + +#### `AndroidBootImageFormatDescriptor` + +Android boot image — `boot.img` / `recovery.img` (magic `ANDROID!`, header v0–v4) and `vendor_boot.img` (magic `VNDRBOOT`, header v3–v4) — as a pseudo-archive whose members are the components `unpack_bootimg` extracts, under the names it gives them: `kernel`, `ramdisk`, `second`, `recovery_dtbo`, `dtb`, `boot_signature`, `vendor_ramdisk` / `vendor_ramdiskNN`, `bootconfig`. `metadata.ini` renders the header; bytes after the last component (partition padding, an AVB footer) surface as `trailer.bin`. + +Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IFormatDescriptor`, `ISyntheticEntryNames`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `AndroidBootImageFormatDescriptor` | `AndroidBootImageFormatDescriptor()` | | +| `BootSignature` | `const string BootSignature` | | +| `Bootconfig` | `const string Bootconfig` | | +| `Dtb` | `const string Dtb` | | +| `FullImage` | `const string FullImage` | | +| `Kernel` | `const string Kernel` | | +| `Ramdisk` | `const string Ramdisk` | | +| `RecoveryDtbo` | `const string RecoveryDtbo` | | +| `Second` | `const string Second` | | +| `Trailer` | `const string Trailer` | | +| `VendorRamdisk` | `const string VendorRamdisk` | | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `SyntheticEntryNames` | `IReadOnlySet SyntheticEntryNames { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Builds an image from component files named as `unpack_bootimg` names them, with the header fields taken from a `metadata.ini` among the inputs (the rendering this descriptor lists) and `mkbootimg`'s defaults for every field it does not set. A single input under any other name is the payload a boot image exists to carry and becomes the kernel, as `mkbootimg --kernel` would take it; more than one such input is refused. | +| `Entries` | `static List Entries(byte[] data)` | The members of `data`; an image that does not parse lists as itself plus the reason. | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Purge` | `void Purge(Stream archive)` | Empties every component and the trailer, keeping the header fields: a valid image with nothing in it. | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | | + +#### `AndroidBootImageMetadata` + +The `metadata.ini` rendering of an Android boot / vendor_boot header, and the reverse: applying an edited rendering back onto an image. A key is applied only when its value differs from what the image itself renders, so handing back an untouched rendering changes nothing — not even the bytes past the NUL of a string field. + +| Member | Signature | Summary | +| --- | --- | --- | +| `BootSection` | `const string BootSection` | | +| `VendorSection` | `const string VendorSection` | | +| `Apply` | `static void Apply(AndroidBootImage img, IReadOnlyDictionary ini)` | Applies every key of `ini` whose value differs from the image's own rendering. | +| `NewImage` | `static AndroidBootImage NewImage(IReadOnlyDictionary ini, bool vendorHint)` | A fresh image of the kind and header version `ini` names (boot v0 when it names none), carrying `mkbootimg`'s defaults: base 0x10000000 with its standard offsets, 2048-byte pages, and the header size of the version. | +| `RamdiskEntryName` | `static string RamdiskEntryName(int index)` | | +| `RenderPartial` | `static byte[] RenderPartial(string error)` | | +| `Render` | `static byte[] Render(AndroidBootImage img)` | | + +#### `VendorRamdisk` + +One fragment of a vendor_boot ramdisk table (header v4) or the single vendor ramdisk (v3). + +| Member | Signature | Summary | +| --- | --- | --- | +| `VendorRamdisk` | `VendorRamdisk()` | | +| `BoardId` | `uint[] BoardId { get; set; }` | | +| `Data` | `byte[] Data { get; set; }` | | +| `Name` | `byte[] Name { get; set; }` | The 32-byte NUL-padded name field, kept raw. | +| `Type` | `uint Type { get; set; }` | | + +### Namespace `FileFormat.AndroidVbmeta` + +[`AndroidVbmetaFormatDescriptor`](#androidvbmetaformatdescriptor) · [`AvbDescriptor`](#avbdescriptor) · [`AvbDescriptorText`](#avbdescriptortext) · [`AvbVbmeta`](#avbvbmeta) + +#### `AndroidVbmetaFormatDescriptor` + +Android Verified Boot 2.0 `vbmeta` image (magic `AVB0`) as a pseudo-archive: `metadata.ini` renders the header, the verification result and every descriptor's fields; each descriptor is a member under `descriptors/` as its exact encoded bytes; the public key, its metadata and the signature are members of their own. + +Implements `IArchiveCreatable`, `IArchiveFormatOperations`, `IArchiveModifiable`, `IArchivePurgeable`, `IArchiveWriteConstraints`, `IFormatDescriptor`, `ISyntheticEntryNames`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `AndroidVbmetaFormatDescriptor` | `AndroidVbmetaFormatDescriptor()` | | +| `DescriptorFolder` | `const string DescriptorFolder` | | +| `FullImage` | `const string FullImage` | | +| `PublicKeyMetadataName` | `const string PublicKeyMetadataName` | | +| `PublicKeyName` | `const string PublicKeyName` | | +| `SignatureName` | `const string SignatureName` | | +| `TrailerName` | `const string TrailerName` | | +| `AcceptedInputsDescription` | `string AcceptedInputsDescription { get; }` | | +| `Capabilities` | `FormatCapabilities Capabilities { get; }` | | +| `Category` | `FormatCategory Category { get; }` | | +| `CompoundExtensions` | `IReadOnlyList CompoundExtensions { get; }` | | +| `DefaultExtension` | `string DefaultExtension { get; }` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Extensions` | `IReadOnlyList Extensions { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `MagicSignatures` | `IReadOnlyList MagicSignatures { get; }` | | +| `MaxTotalArchiveSize` | `long? MaxTotalArchiveSize { get; }` | | +| `Methods` | `IReadOnlyList Methods { get; }` | | +| `SyntheticEntryNames` | `IReadOnlySet SyntheticEntryNames { get; }` | | +| `TarCompressionFormatId` | `string TarCompressionFormatId { get; }` | | +| `Add` | `void Add(Stream archive, IReadOnlyList inputs)` | | +| `CanAccept` | `bool CanAccept(ArchiveInputInfo input, out string reason)` | The members `Create` writes. A vbmeta holds structured AVB descriptors, not files, so an arbitrary blob has nowhere to go; a public key or signature needs the private key. | +| `Create` | `void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options)` | Writes an unsigned vbmeta image from descriptor blobs (`descriptors/NN_*.bin`, in `NN` order), an optional `public_key_metadata.bin`, an optional `trailer.bin` and the header fields of a `metadata.ini` — required libavb version, rollback index and location, flags and release string. A public key or signature among the inputs is refused: signing needs the private key. | +| `Entries` | `static List Entries(byte[] data)` | | +| `Extract` | `void Extract(Stream stream, string outputDir, string password, string[] files)` | | +| `List` | `List List(Stream stream, string password)` | | +| `Purge` | `void Purge(Stream archive)` | | +| `Remove` | `void Remove(Stream archive, string[] entryNames)` | | +| `RenderMetadata` | `static byte[] RenderMetadata(AvbVbmeta v)` | | + +#### `AvbDescriptor` + +One AVB descriptor, kept as its exact encoded bytes (16-byte tag/length header, body, padding). + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `AvbDescriptor` | `AvbDescriptor(ulong Tag, byte[] Encoded)` | One AVB descriptor, kept as its exact encoded bytes (16-byte tag/length header, body, padding). | +| `ChainPartition` | `const ulong ChainPartition` | | +| `Hash` | `const ulong Hash` | | +| `Hashtree` | `const ulong Hashtree` | | +| `KernelCmdline` | `const ulong KernelCmdline` | | +| `Property` | `const ulong Property` | | +| `Body` | `ReadOnlySpan Body { get; }` | | +| `Encoded` | `byte[] Encoded { get; init; }` | | +| `Kind` | `string Kind { get; }` | | +| `Tag` | `ulong Tag { get; init; }` | | + +#### `AvbDescriptorText` + +Human-readable renderings of AVB descriptors: the fields `avbtool info_image` prints, as `key = value` lines, and a short name for the descriptor's entry. Rendering never throws — a descriptor whose body does not match its declared lengths is reported as malformed. + +| Member | Signature | Summary | +| --- | --- | --- | +| `EntryName` | `static string EntryName(int index, AvbDescriptor d)` | | +| `Render` | `static void Render(StringBuilder sb, AvbDescriptor d)` | | +| `Subject` | `static string Subject(AvbDescriptor d)` | The partition name, property key, or nothing — whatever names the descriptor's subject. | + +#### `AvbVbmeta` + +An Android Verified Boot 2.0 `vbmeta` image (magic `AVB0`): the 256-byte header, the authentication block (hash, signature) and the auxiliary block (descriptors, public key, public key metadata), parsed and verified, and written back the way `avbtool make_vbmeta_image` lays one out. + +| Member | Signature | Summary | +| --- | --- | --- | +| `AvbVbmeta` | `AvbVbmeta()` | | +| `HeaderSize` | `const int HeaderSize` | | +| `ReleaseStringSize` | `const int ReleaseStringSize` | | +| `ReservedSize` | `const int ReservedSize` | | +| `AlgorithmType` | `uint AlgorithmType { get; set; }` | | +| `Descriptors` | `List Descriptors { get; set; }` | | +| `Flags` | `uint Flags { get; set; }` | | +| `HashValid` | `bool? HashValid { get; }` | Null when the algorithm carries no hash. | +| `Hash` | `byte[] Hash { get; set; }` | | +| `IsCanonical` | `bool IsCanonical { get; }` | True when the image is laid out as avbtool lays it out, so a rewrite reproduces it. | +| `NonCanonicalReason` | `string NonCanonicalReason { get; }` | | +| `PublicKeyMetadata` | `byte[] PublicKeyMetadata { get; set; }` | | +| `PublicKey` | `byte[] PublicKey { get; set; }` | | +| `ReleaseString` | `byte[] ReleaseString { get; set; }` | The 48-byte release-string field, raw (avbtool writes 47 bytes plus a NUL). | +| `Reserved` | `byte[] Reserved { get; set; }` | | +| `RollbackIndexLocation` | `uint RollbackIndexLocation { get; set; }` | | +| `RollbackIndex` | `ulong RollbackIndex { get; set; }` | | +| `SignatureValid` | `bool? SignatureValid { get; }` | Null when the algorithm carries no signature this reader can check. | +| `Signature` | `byte[] Signature { get; set; }` | | +| `Trailer` | `byte[] Trailer { get; set; }` | Bytes after the vbmeta blob (partition padding). | +| `VersionMajor` | `uint VersionMajor { get; set; }` | | +| `VersionMinor` | `uint VersionMinor { get; set; }` | | +| `AlgorithmName` | `static string AlgorithmName(uint type)` | | +| `FromEncoded` | `static AvbDescriptor FromEncoded(byte[] encoded)` | | +| `Parse` | `static AvbVbmeta Parse(ReadOnlySpan d)` | | +| `ReleaseText` | `static string ReleaseText(byte[] field)` | | +| `TryRsaKey` | `static RSAParameters? TryRsaKey(ReadOnlySpan key)` | AVB's public key encoding: big-endian bit count and n0inv, then the modulus and R², each bit-count/8 bytes. Exponent 65537. | +| `Validate` | `static void Validate(AvbDescriptor desc)` | Checks an encoded descriptor's own length field, so a malformed blob is refused rather than written. | +| `Write` | `void Write(Stream output)` | Writes the vbmeta blob as avbtool's `_generate_vbmeta_blob` does, then the trailer. Only the unsigned `NONE` algorithm can be written: anything else needs the private key. | + ### Namespace `FileFormat.BinCue` [`BinCueEntry`](#bincueentry) · [`BinCueFormatDescriptor`](#bincueformatdescriptor) · [`BinCueInPlaceModifier`](#bincueinplacemodifier) · [`BinCueInPlaceModifier.SectorGeometry`](#bincueinplacemodifiersectorgeometry) · [`BinCueLayoutMap`](#bincuelayoutmap) · [`BinCueReader`](#bincuereader) @@ -603,6 +848,56 @@ Writer for EnCase Expert Witness Format (EWF / .E01) forensic images. Produces a | `Notes` | `string Notes { get; init; }` | Notes recorded in the acquisition header. | | `Build` | `byte[] Build(ReadOnlySpan media)` | Builds a single-segment E01 image from the supplied raw media bytes. | +### Namespace `FileFormat.FirmwareCommon` + +[`EmbeddedPayload`](#embeddedpayload) · [`FirmwareArchive`](#firmwarearchive) · [`FirmwareCrc32`](#firmwarecrc32) · [`FirmwareEntry`](#firmwareentry) + +#### `EmbeddedPayload` + +Recognises and unpacks a compressed payload embedded in a firmware image by the magic of its stream format, using this repository's own stream codecs through the registry. A payload is only reported as decompressed when the codec decodes it without error, so nothing is ever guessed into existence. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Find` | `static ValueTuple? Find(ReadOnlyMemory data, int start, Func accept, int maxCandidates = 64)` | Scans `data` from `start` for the first known stream magic whose codec decodes and whose output satisfies `accept`. | +| `Identify` | `static ValueTuple? Identify(ReadOnlySpan data)` | The stream format whose magic starts `data`, or null. | +| `TryDecompress` | `static byte[] TryDecompress(string formatId, ReadOnlyMemory data, long limit = 1073741824)` | Decodes `data` with the codec `formatId` names, giving up past `limit` output bytes. Null when the codec is not registered or rejects the stream. | + +#### `FirmwareArchive` + +The list / extract plumbing every firmware pseudo-archive shares: a firmware image is parsed into an ordered set of named slices, and listing and extracting both walk that set. + +| Member | Signature | Summary | +| --- | --- | --- | +| `MetadataName` | `const string MetadataName` | | +| `CString` | `static string CString(ReadOnlySpan field)` | A NUL-terminated field as text: everything before the first NUL, Latin-1 so no byte is lost. | +| `Extract` | `static void Extract(IReadOnlyList entries, string outputDir, string[] files)` | | +| `Hex` | `static string Hex(ulong value)` | | +| `List` | `static List List(IReadOnlyList entries)` | | +| `ParseIni` | `static Dictionary ParseIni(ReadOnlySpan ini)` | Parses the `key = value` lines of an INI rendering into section-qualified keys (`section.key`); keys before any section are unqualified. The value keeps its exact text after the single blank that follows the `=`. | +| `ReadAll` | `static byte[] ReadAll(Stream stream)` | | +| `TryParseNumber` | `static bool TryParseNumber(string text, out ulong value)` | | + +#### `FirmwareCrc32` + +The reflected CRC-32 (polynomial 0xEDB88320) firmware headers use, exposed raw: initial value and final XOR are the caller's, because TRX for one stores the register without the final inversion. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Compute` | `static uint Compute(ReadOnlySpan data)` | The standard CRC-32 (zlib, IEEE 802.3). | +| `Update` | `static uint Update(uint crc, ReadOnlySpan data)` | | + +#### `FirmwareEntry` + +One member a firmware container exposes: a slice of the image or a rendering of it. + +Implements `IEquatable`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `FirmwareEntry` | `FirmwareEntry(string Name, ReadOnlyMemory Data)` | One member a firmware container exposes: a slice of the image or a rendering of it. | +| `Data` | `ReadOnlyMemory Data { get; init; }` | | +| `Name` | `string Name { get; init; }` | | + ### Namespace `FileFormat.FirmwareHex` [`FirmwareHexWriter`](#firmwarehexwriter) · [`FirmwareImage`](#firmwareimage) · [`IntelHexFormatDescriptor`](#intelhexformatdescriptor) · [`IntelHexReader`](#intelhexreader) · [`SRecordReader`](#srecordreader) · [`TiTxtFormatDescriptor`](#titxtformatdescriptor) · [`TiTxtReader`](#titxtreader) diff --git a/docs/LICENCE-PROVENANCE.md b/docs/LICENCE-PROVENANCE.md index e940b18c7..3134fd108 100644 --- a/docs/LICENCE-PROVENANCE.md +++ b/docs/LICENCE-PROVENANCE.md @@ -31,7 +31,7 @@ package carries; everything else keeps a `THIRD-PARTY-NOTICE..txt` beside | Class | Upstream rows | Notes | |---|---|---| -| A | 16 | | +| A | 18 | | | B | 17 | FFmpeg alone covers 38 codec folders | | C | 4 | specifications: `docs/CVF-CODECS-ON-DISK.md` (rewrite already under way), `docs/G722-SPEC.md`, `docs/IT-SAMPLE-COMPRESSION-SPEC.md`, `docs/FDS-EXPANSION-SOUND-SPEC.md` | | R | 7 | | @@ -71,6 +71,8 @@ package carries; everything else keeps a `THIRD-PARTY-NOTICE..txt` beside | Claim site | Upstream | Licence | Where the notice is | |---|---|---|---| +| `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImage.cs`, `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImageMetadata.cs` | AOSP mkbootimg `bootimg.h`, `mkbootimg.py` and `unpack_bootimg.py`, revision `d2bb0af5ba6d3198a3e99529c97eda1be0b5a093` | BSD-3-Clause and Apache-2.0 | `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/THIRD-PARTY-NOTICE.mkbootimg.txt` | +| `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbVbmeta.cs`, `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbDescriptorText.cs` | AOSP Android Verified Boot headers and `avbtool.py`, revision `c5066a96caa7bf4150c0a8cc8cc14ab81733fdc7` | MIT | `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/THIRD-PARTY-NOTICE.avb.txt` | | `Compression.Core/Dictionary/Lzbitmap/` | libzbitmap (Ernesto A. Fernández, Corellium) | MIT | `THIRD-PARTY-NOTICE.libzbitmap.txt` | | `Compression.Core/Dictionary/Csc/` | libcsc (Fu Siyuan) | Unlicense | `THIRD-PARTY-NOTICE.libcsc.txt` | | `Compression.Core/Dictionary/Arj/ArjFastestDecoder.cs` | XADMaster `XADARJFastestHandle.m` (Dag Ågren, MacPaw), revision `7cb9ee0abbb163f261e4cb74501e15067032319c` | LGPL-2.1-or-later | `Compression.Core/Dictionary/Arj/THIRD-PARTY-NOTICE.XADMaster.txt` | diff --git a/docs/catalogue/our-registry.csv b/docs/catalogue/our-registry.csv index b9f718474..077b4323f 100644 --- a/docs/catalogue/our-registry.csv +++ b/docs/catalogue/our-registry.csv @@ -26,9 +26,11 @@ format,Ampk,AMPK (Amiga Pack),Archive,,.ampk,Y,Y,Y,Y format,Amr,3GPP AMR,Audio,,.amr;.awb;.3ga,Y,Y,Y, format,AmrNb,AMR-NB,Audio,,.amr,,,, format,AmrWb,AMR-WB,Audio,,.amr,,,, +format,AndroidBootImage,Android boot image,Archive,,.bootimg,Y,Y,Y,Y format,AndroidBundle,Android App Bundle / split-APK set,Archive,,.aab;.apks,Y,Y,Y,Y format,AndroidOta,Android OTA payload,Archive,,,Y,Y,Y, format,AndroidSparse,Android Sparse Image,Archive,,.simg;.sparse,Y,Y,Y, +format,AndroidVbmeta,Android Verified Boot (vbmeta),Archive,,.avb,Y,Y,Y,Y format,Ani,ANI (animated cursor),Archive,,.ani,Y,Y,Y, format,Aomei,AOMEI Backupper Image (ADI/AFI),Archive,,.adi;.afi,Y,Y,Y,Y format,Apc,CRYO APC,Audio,,.apc,Y,Y,Y, diff --git a/docs/catalogue/reference-formats.csv b/docs/catalogue/reference-formats.csv index c5e52df48..a0d236b05 100644 --- a/docs/catalogue/reference-formats.csv +++ b/docs/catalogue/reference-formats.csv @@ -258,7 +258,7 @@ VMDK,vmdk,disk-image,R,,,,R,,Y,3,https://developer.broadcom.com/,Vmdk,Y,Y,Y,OK,, VirtualBox VDI,vdi,disk-image,R,,,,R,fmt/726,Y,4,https://www.virtualbox.org/,Vdi,Y,Y,Y,OK,, QCOW / QCOW2,qcow;qcow2;qcow2c,disk-image,R,,,D,,,Y,3,https://gitlab.com/qemu-project/qemu/-/blob/master/docs/interop/qcow2.txt,Qcow2,Y,Y,Y,PARTIAL,,"no encryption, backing files, zstd clusters or extended L2 (Qcow2Structures.cs)" Android sparse image,simg;sparse,disk-image,R,,,R,,,,2,https://android.googlesource.com/platform/system/core/+/master/libsparse/sparse_format.h,AndroidSparse,Y,Y,,OK,, -Android boot image,img;bootimg,firmware,,,,D,R,,,2,,,,,,MISSING,,https://source.android.com/docs/core/architecture/bootloader/boot-image-header +Android boot image,img;bootimg,firmware,,,,D,R,,,2,https://source.android.com/docs/core/architecture/bootloader/boot-image-header,AndroidBootImage,Y,Y,Y,OK,,mkbootimg/unpack_bootimg byte-identical oracle; vendor_boot v3/v4 included EnCase / Expert Witness (E01),e01;ex01;l01,disk-image,,,,,,fmt/803,Y,2,https://github.com/libyal/libewf/blob/main/documentation/Expert%20Witness%20Compression%20Format%20(EWF).asciidoc,Ewf,Y,Y,Y,PARTIAL,,"single segment only (no .E02+), no L01 logical evidence, no Ex01 (EwfReader.cs)" AFF (Advanced Forensic Format v1-3),aff;afd;afm,disk-image,,,,,,,Y,1,,,,,,MISSING,, Nero image (NRG),nrg,disk-image,,,,,R,fmt/1743,Y,3,,Nrg,Y,Y,Y,OK,, @@ -306,7 +306,7 @@ LhF,lhf,archive,,,R,,,,,1,,LhF,Y,Y,Y,OK,, MDC (Amiga),mdc,archive,,,R,,,,,1,,,,,,MISSING,, MXM-SimpleArc,,archive,,,R,,,,,1,,,,,,MISSING,, UEFI firmware volume / capsule,fv;fd;cap;bios;uefi;uefif,firmware,R,,,R,,,,2,https://uefi.org/specifications,UefiFv,Y,Y,Y,OK,, -Android Verified Boot (vbmeta),img;avb,firmware,R,,,,,,,1,,,,,,MISSING,, +Android Verified Boot (vbmeta),img;avb,firmware,R,,,,,,,1,https://android.googlesource.com/platform/external/avb/+/main/README.md,AndroidVbmeta,Y,Y,Y,OK,,hash + RSA signature verified on read; unsigned (NONE) images written byte-identical to avbtool; signed images read-only Device tree blob,dtb;dtbo;fdt,firmware,,,,R,,,,1,https://www.devicetree.org/specifications/,Dtb,Y,Y,Y,OK,, U-Boot uImage,uimage,firmware,,,,R,,,,1,,UImage,Y,Y,Y,OK,, TRX firmware,trx,firmware,,,,R,,,,1,,,,,,MISSING,,