From 13b615097a7a892f0ac37ea63cfd7a9a9eaaea00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=B1=EC=A3=BC?= Date: Wed, 28 Oct 2026 00:44:51 +0900 Subject: [PATCH] [AUTOMATED] Add standalone thin Mach-O DWARF reader --- decbench/utils/macho_dwarf.py | 116 +++++++++++++++++++++++ docs/macho-dwarf-reader.md | 38 ++++++++ tests/test_macho_dwarf_reader.py | 154 +++++++++++++++++++++++++++++++ 3 files changed, 308 insertions(+) create mode 100644 decbench/utils/macho_dwarf.py create mode 100644 docs/macho-dwarf-reader.md create mode 100644 tests/test_macho_dwarf_reader.py diff --git a/decbench/utils/macho_dwarf.py b/decbench/utils/macho_dwarf.py new file mode 100644 index 0000000..45a6df6 --- /dev/null +++ b/decbench/utils/macho_dwarf.py @@ -0,0 +1,116 @@ +"""Read thin Mach-O DWARF without changing the benchmark's existing metrics. + +The binary and its companion are supplied by the evaluator, never the producer. +Universal binaries and ambiguous/missing debug companions are not resolved. +""" + +import struct +from pathlib import Path + +from decbench.utils.binfmt import _DWARF_SECS, _build_dwarfinfo + +_MAGICS = { + b"\xce\xfa\xed\xfe": ("<", 32), + b"\xcf\xfa\xed\xfe": ("<", 64), + b"\xfe\xed\xfa\xce": (">", 32), + b"\xfe\xed\xfa\xcf": (">", 64), +} +_MACHINES = { + 7: "x86", + 0x1000007: "x64", + 12: "ARM", + 0x100000C: "AArch64", + 18: "PowerPC", + 0x1000012: "PowerPC64", +} + + +def _thin_header(path: Path): + """Return endian, address size, and machine only for a known thin header.""" + try: + with path.open("rb") as stream: + header = stream.read(16) + if len(header) != 16 or header[:4] not in _MAGICS: + return None + endian, bits = _MAGICS[header[:4]] + cpu, subtype, file_type = struct.unpack(endian + "III", header[4:16]) + machine = _MACHINES.get(cpu) + if machine is None or bool(cpu & 0x01000000) != (bits == 64): + return None + return endian, bits // 8, machine, cpu, subtype, file_type + except (OSError, struct.error): + return None + + +def _image(path: Path): + if _thin_header(path) is None: + return None + try: + import lief + + images = lief.MachO.parse(str(path)) + return images[0] if images is not None and len(images) == 1 else None + except (OSError, RuntimeError, ValueError, TypeError): + return None + + +def _sections(image): + sections = {} + for section in image.sections: + if section.segment_name != "__DWARF" or not section.name.startswith("__debug_"): + continue + name = "." + section.name[2:] + if name == ".debug_str_offs": + name = ".debug_str_offsets" + if name in _DWARF_SECS: + # Duplicate sections cannot safely be assigned a unique oracle. + if name in sections: + return None + sections[name] = bytes(section.content) + return sections + + +def _has_units(sections): + return bool(sections and sections.get(".debug_info") and sections.get(".debug_abbrev")) + + +def macho_dwarf_info(path: Path, *, debug_path: Path | None = None): + """Read embedded DWARF or a thin UUID/architecture-matched dSYM. + + Explicit ``debug_path`` selects a companion and is never silently ignored. + Otherwise embedded DWARF is preferred and the canonical adjacent dSYM is + used only if embedded sections are absent. Returns None when unsupported or + identity cannot be established. Does not affect ``binfmt.dwarf_info``. + """ + header = _thin_header(path) + image = _image(path) + if header is None or header[-1] not in (2, 6, 8) or image is None: + return None + sections = _sections(image) if debug_path is None else {} + if sections is None: + return None + if not _has_units(sections): + companion = debug_path if debug_path is not None else path.with_name(path.name + ".dSYM") + if companion.is_dir(): + companion = companion / "Contents" / "Resources" / "DWARF" / path.name + companion_header = _thin_header(companion) + if ( + companion_header is None + or companion_header[:-1] != header[:-1] + or companion_header[-1] != 10 + ): + return None + debug = _image(companion) + if debug is None or not image.has_uuid or not debug.has_uuid: + return None + image_uuid = bytes(image.uuid.uuid) + debug_uuid = bytes(debug.uuid.uuid) + if len(image_uuid) != 16 or not any(image_uuid) or image_uuid != debug_uuid: + return None + sections = _sections(debug) + if not _has_units(sections): + return None + endian, address_size, machine, _, _, _ = header + return _build_dwarfinfo( + sections, little_endian=endian == "<", addr_size=address_size, march=machine + ) diff --git a/docs/macho-dwarf-reader.md b/docs/macho-dwarf-reader.md new file mode 100644 index 0000000..62f32fa --- /dev/null +++ b/docs/macho-dwarf-reader.md @@ -0,0 +1,38 @@ +# Experimental thin Mach-O DWARF reader + +`decbench.utils.macho_dwarf.macho_dwarf_info` is a low-level reader for a thin +Mach-O linked image and its embedded DWARF or a UUID/CPU/subtype-matched dSYM. +Linked inputs are limited to executable, dynamic library, and bundle images; +external debug companions must be Mach-O dSYM files. It is +not wired into `binfmt.dwarf_info`, decompiler adapters, metrics, cache keys, +the result pipeline, or official rankings. This patch is a first independently +reviewable component; the existing C and ELF/PE evaluation path remains in +place. Integrating it requires a separate change and validation of source and +binary function identity, debug evidence lifetime, and cache behavior. + +An explicitly supplied dSYM is always checked. If none is supplied, complete +embedded debug sections take precedence, otherwise the canonical adjacent +`.dSYM` bundle is considered. Missing, universal, architecture/subtype +or UUID-mismatched inputs return `None`; duplicate DWARF sections also abstain. +UUID matching is necessary identity evidence, not proof that a binary's code +sections have not been modified. The returned DWARF section buffers are owned +in memory. Full DWARF semantic validity, Mach-O function attribution, split +DWARF, universal binary selection and executable correctness are not established +by this API. + +## Fixed-project diagnostic + +On macOS arm64, the O0 OpenCC stripped image and dSYM from the unofficial C++ +qualification dataset revision `07dd95dbe6214b8a78fc1e4a55961ab515986489` +were fetched and verified against that revision's SHA-256 manifest. The reader +found 33 compilation units and 8,911 subprogram DIEs with `DW_AT_low_pc`; +these are raw DIEs, not 8,911 unique or source-owned functions. Supplying the +separately built O2 OpenCC dSYM to the O0 image returned `None`. This is a +reader diagnostic, not a metric or benchmark result. Source information is +used only as evaluator-side evidence; the dataset itself remains marked +`qualified=false` and `scoring_ready=false`. + +Focused tests build a real C executable and dSYM on Darwin, verify UUID mismatch +rejection and retain the decoded data after removal of the companion. Other tests +exercise malformed headers, subtype mismatch, UUID absence and duplicate debug +sections. Outside Darwin, the native-tool test is skipped. diff --git a/tests/test_macho_dwarf_reader.py b/tests/test_macho_dwarf_reader.py new file mode 100644 index 0000000..e8562e5 --- /dev/null +++ b/tests/test_macho_dwarf_reader.py @@ -0,0 +1,154 @@ +"""Tests for the unconnected thin Mach-O debug-reader building block.""" + +import shutil +import struct +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from decbench.utils import binfmt, macho_dwarf + + +def test_rejects_truncated_universal_and_unknown_mach_headers(tmp_path: Path) -> None: + path = tmp_path / "input" + for data in ( + b"", + b"\xcf\xfa\xed\xfe", + b"\xca\xfe\xba\xbe" + b"\0" * 16, + b"\xcf\xfa\xed\xfe" + struct.pack(" None: + section = SimpleNamespace(segment_name="__DWARF", name="__debug_info", content=[1]) + assert macho_dwarf._sections(SimpleNamespace(sections=[section, section])) is None + + +def test_rejects_missing_or_mismatched_uuid( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + image = tmp_path / "input" + debug = tmp_path / "debug" + # Sufficient for the local header gate; _image is replaced with controlled objects. + header = b"\xcf\xfa\xed\xfe" + struct.pack(" None: + image = tmp_path / "input" + debug = tmp_path / "debug" + image.write_bytes(b"\xcf\xfa\xed\xfe" + struct.pack(" None: + image, companion = tmp_path / "image", tmp_path / "debug" + uuid = bytes(range(16)) + + def section(name): + return SimpleNamespace(segment_name="__DWARF", name=name, content=[1]) + + main = SimpleNamespace(sections=[], has_uuid=True, uuid=SimpleNamespace(uuid=uuid)) + debug = SimpleNamespace( + sections=[section("__debug_info"), section("__debug_abbrev")], + has_uuid=True, + uuid=SimpleNamespace(uuid=uuid), + ) + monkeypatch.setattr(macho_dwarf, "_image", lambda path: main if path == image else debug) + + def header(file_type): + return b"\xcf\xfa\xed\xfe" + struct.pack(" None: + cc = shutil.which("clang") + dsymutil = shutil.which("dsymutil") + strip = shutil.which("strip") + if not all((cc, dsymutil, strip)): + pytest.skip("clang, dsymutil and strip are required") + source = tmp_path / "unit.c" + source.write_text("int value(int x) { return x + 3; }\nint main(void) { return value(1); }\n") + obj, image = tmp_path / "unit.o", tmp_path / "unit" + subprocess.run( + [cc, "-g", "-O0", "-gdwarf-4", "-c", str(source), "-o", str(obj)], + check=True, + capture_output=True, + ) + subprocess.run([cc, str(obj), "-o", str(image)], check=True, capture_output=True) + bundle = tmp_path / "unit.dSYM" + subprocess.run([dsymutil, str(image), "-o", str(bundle)], check=True, capture_output=True) + # A real dSYM from a different linked image must never be used as oracle. + other_source, other_object, other_image = ( + tmp_path / "other.c", + tmp_path / "other.o", + tmp_path / "other", + ) + other_source.write_text( + "int value(int x) { return x + 4; }\nint main(void) { return value(2); }\n" + ) + subprocess.run( + [cc, "-g", "-O0", "-gdwarf-4", "-c", str(other_source), "-o", str(other_object)], + check=True, + capture_output=True, + ) + subprocess.run([cc, str(other_object), "-o", str(other_image)], check=True, capture_output=True) + assert ( + macho_dwarf.macho_dwarf_info( + other_image, debug_path=bundle / "Contents" / "Resources" / "DWARF" / "unit" + ) + is None + ) + stripped = tmp_path / "stripped" + subprocess.run([strip, "-S", "-o", str(stripped), str(image)], check=True, capture_output=True) + stripped.replace(image) + dwarf = macho_dwarf.macho_dwarf_info(image) + assert dwarf is not None + shutil.rmtree(bundle) + names = { + die.attributes["DW_AT_name"].value + for cu in dwarf.iter_CUs() + for die in cu.iter_DIEs() + if "DW_AT_name" in die.attributes + } + assert b"value" in names + assert macho_dwarf.macho_dwarf_info(image) is None + # The existing evaluation entry point has not been wired to the new reader. + assert binfmt.dwarf_info(image) is None